From b888236c0a58cc9c8ec2cb0c7cc7ebbcc7ac5c94 Mon Sep 17 00:00:00 2001 From: KonradSzwarc Date: Sat, 23 Oct 2021 12:52:12 +0200 Subject: [PATCH] feat(cli): create command to generate CSV with course participants (#399) --- .gitignore | 1 + packages/cli/package.json | 5 +- .../cli/src/commands/generate-checklists.ts | 11 +++-- .../cli/src/commands/generate-welcome-csv.ts | 45 +++++++++++++++++ .../cli/src/commands/register-participants.ts | 47 +++++++++++------- packages/cli/src/index.ts | 2 + packages/cli/src/shared/array.ts | 11 +++++ packages/cli/src/shared/csv.ts | 6 ++- packages/cli/src/shared/db.ts | 36 +++++--------- packages/cli/src/shared/env.ts | 22 +-------- packages/cli/src/shared/files.ts | 43 ++++++++++++++++ packages/cli/src/shared/models.ts | 49 ++++++++++++++++--- packages/cli/src/shared/object.ts | 34 +++++++++++-- yarn.lock | 48 ++++++++++++++---- 14 files changed, 273 insertions(+), 87 deletions(-) create mode 100644 packages/cli/src/commands/generate-welcome-csv.ts create mode 100644 packages/cli/src/shared/array.ts create mode 100644 packages/cli/src/shared/files.ts diff --git a/.gitignore b/.gitignore index 3c5667f7..aad36db4 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,7 @@ out # Generated packages/ui/src/svg packages/ui/src/icons +packages/cli/output .cache-loader # Database diff --git a/packages/cli/package.json b/packages/cli/package.json index 76535e73..c1896dff 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -12,9 +12,12 @@ "commander": "8.2.0", "csv-parse": "4.16.3", "dotenv": "10.0.0", - "generate-password": "1.6.1", + "json2csv": "5.0.6", "puppeteer": "10.4.0", "reflect-metadata": "0.1.13", "winston": "3.3.3" + }, + "devDependencies": { + "@types/json2csv": "5.0.3" } } diff --git a/packages/cli/src/commands/generate-checklists.ts b/packages/cli/src/commands/generate-checklists.ts index 27d906cc..205023a1 100644 --- a/packages/cli/src/commands/generate-checklists.ts +++ b/packages/cli/src/commands/generate-checklists.ts @@ -53,9 +53,14 @@ export const generateChecklists = (program: Command) => { logger.debug('Iterating through fetched participants'); for (const participant of participants) { - const checklist = await generateProcessStChecklist(participant.name); - - await updateUserById(participant.id, { checklist }); + if (!participant.checklist) { + const name = `${participant.firstName} ${participant.lastName}`; + const checklist = await generateProcessStChecklist(name); + + await updateUserById(participant.id, { checklist }); + } else { + logger.debug('Participant already has a checklist', participant); + } } logger.debug('Iteration through fetched participants finished'); diff --git a/packages/cli/src/commands/generate-welcome-csv.ts b/packages/cli/src/commands/generate-welcome-csv.ts new file mode 100644 index 00000000..80b5389f --- /dev/null +++ b/packages/cli/src/commands/generate-welcome-csv.ts @@ -0,0 +1,45 @@ +import { isInt, isPositive } from 'class-validator'; +import { Argument, Command } from 'commander'; + +import { parseToCsv } from '../shared/csv'; +import { getUsersByRole } from '../shared/db'; +import { createWelcomeCsv } from '../shared/files'; +import { createLogger } from '../shared/logger'; +import { userRoles, WelcomeCsvRow } from '../shared/models'; +import { transformAndValidate } from '../shared/object'; + +const logger = createLogger('generate-welcome-csv'); + +export const generateWelcomeCsv = (program: Command) => { + program + .command('generate-welcome-csv') + .description('Generates CSV file with names, emails and checklist URLs for all participants in the database') + .addArgument(new Argument('', 'Path to the CSV file')) + .action(async (fromId?: string) => { + const startFromId = Number(fromId); + + try { + if (fromId && !(isPositive(startFromId) && isInt(startFromId))) { + throw new Error('fromId parameter must be a positive integer'); + } + + let participants = await getUsersByRole(userRoles.participant); + + if (startFromId) { + participants = participants.filter((p) => p.id >= startFromId); + } + + const welcomeCsvRows = await Promise.all(participants.map(transformAndValidate(WelcomeCsvRow))); + + logger.debug('Parsing welcome CSV objects to CSV format', welcomeCsvRows); + + const csv = parseToCsv(welcomeCsvRows); + + logger.debug('Creating welcome CSVfile', csv); + + await createWelcomeCsv(csv); + } catch (ex) { + logger.error(ex); + } + }); +}; diff --git a/packages/cli/src/commands/register-participants.ts b/packages/cli/src/commands/register-participants.ts index d5dc5611..545532c4 100644 --- a/packages/cli/src/commands/register-participants.ts +++ b/packages/cli/src/commands/register-participants.ts @@ -1,11 +1,12 @@ import { Argument, Command } from 'commander'; +import { asyncFilter, removeDuplicatesForProperty } from '../shared/array'; import { getCsvContent } from '../shared/csv'; -import { insertUsers, register } from '../shared/db'; +import { getUsersByRole, insertUsers } from '../shared/db'; import { validateEnv } from '../shared/env'; import { createLogger } from '../shared/logger'; -import { ParticipantCsvRow, RegisterDTO, User, userRoles } from '../shared/models'; -import { transformToMatchClass } from '../shared/object'; +import { CreateUserDTO, ParticipantCsvRow, userRoles } from '../shared/models'; +import { filterInvalid, transformAndValidate, transformToMatchClass } from '../shared/object'; const logger = createLogger('register-participants'); @@ -20,27 +21,37 @@ export const registerParticipants = (program: Command) => { const rows = await getCsvContent(csvPath); const participantsRows = await Promise.all(rows.map(transformToMatchClass(ParticipantCsvRow))); + const correctParticipantsRows = await asyncFilter(participantsRows, filterInvalid(ParticipantCsvRow)); - const participants: User[] = []; + const currentParticipants = await getUsersByRole(userRoles.participant); + const currentParticipantsEmails = currentParticipants.map(({ email }) => email); - logger.debug('Iterating through parsed rows'); + logger.debug('Filtering emails that are already added to the database'); - for (const { email, firstName, lastName } of participantsRows) { - const registerDto = await transformToMatchClass(RegisterDTO)({ email }); - const userId = await register(registerDto); - const participant = await transformToMatchClass(User)({ - ...registerDto, - id: userId, - name: `${firstName} ${lastName}`, - role: userRoles.participant, - }); + const participantsRowsToAdd = correctParticipantsRows.filter(({ email }) => { + if (currentParticipantsEmails.includes(email)) { + logger.debug(`Participant with email ${email} already exists in the database`); - participants.push(participant); - } + return false; + } - logger.debug('Iteration through parsed rows finished'); + return true; + }); - await insertUsers(participants); + logger.debug('Mapping ParticipantCsvRows to CreateUserDTOs'); + + const createUserDTOs = await Promise.all( + participantsRowsToAdd.map(({ email, firstName, lastName }) => + transformAndValidate(CreateUserDTO)({ + email, + firstName, + lastName, + role: userRoles.participant, + }), + ), + ); + + await insertUsers(removeDuplicatesForProperty(createUserDTOs, 'email')); } catch (ex) { logger.error(ex); } diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index e74f60b1..ef19aa19 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -3,6 +3,7 @@ import 'reflect-metadata'; import { Command } from 'commander'; import { generateChecklists } from './commands/generate-checklists'; +import { generateWelcomeCsv } from './commands/generate-welcome-csv'; import { registerParticipants } from './commands/register-participants'; const program = new Command(); @@ -11,5 +12,6 @@ program.version('0.0.0'); registerParticipants(program); generateChecklists(program); +generateWelcomeCsv(program); program.parse(); diff --git a/packages/cli/src/shared/array.ts b/packages/cli/src/shared/array.ts new file mode 100644 index 00000000..5fa8c712 --- /dev/null +++ b/packages/cli/src/shared/array.ts @@ -0,0 +1,11 @@ +export const asyncFilter = async ( + arr: Item[], + predicate: (item: Item, index: number, arr: Item[]) => Promise, +) => { + const results = await Promise.all(arr.map(predicate)); + + return arr.filter((_value, i) => results[i]); +}; + +export const removeDuplicatesForProperty = (arr: Item[], property: keyof Item) => + arr.filter((value, i, array) => array.findIndex((t) => t[property] === value[property]) === i); diff --git a/packages/cli/src/shared/csv.ts b/packages/cli/src/shared/csv.ts index 9c84ec32..459bd62a 100644 --- a/packages/cli/src/shared/csv.ts +++ b/packages/cli/src/shared/csv.ts @@ -1,8 +1,10 @@ -import parse from 'csv-parse/lib/sync'; +import parseFromCsv from 'csv-parse/lib/sync'; import { readFile } from 'fs/promises'; import { createLogger } from './logger'; +export { parse as parseToCsv } from 'json2csv'; + const logger = createLogger('CSV Utils'); export const getCsvContent = async (csvPath: string) => { @@ -12,7 +14,7 @@ export const getCsvContent = async (csvPath: string) => { logger.debug('parsing content of the CSV file'); - const parsedContent: Record[] = parse(content, { columns: true }); + const parsedContent: Record[] = parseFromCsv(content, { columns: true }); logger.debug('CSV file content parsed successfully'); diff --git a/packages/cli/src/shared/db.ts b/packages/cli/src/shared/db.ts index 574d4fdd..558ca01b 100644 --- a/packages/cli/src/shared/db.ts +++ b/packages/cli/src/shared/db.ts @@ -1,9 +1,8 @@ import { createClient } from '@supabase/supabase-js'; -import { generate } from 'generate-password'; import { env } from './env'; import { createLogger } from './logger'; -import { RegisterDTO, Role, User } from './models'; +import { CreateUserDTO, Role, User } from './models'; const logger = createLogger('DB Utils'); @@ -11,29 +10,12 @@ const USERS_TABLE_NAME = 'users'; const supabase = createClient(env.SUPABASE_URL, env.SUPABASE_ANON_KEY); -export const register = async (registerDto: RegisterDTO): Promise => { - logger.debug(`Registering user with email ${registerDto.email}`); - - const { user, error } = await supabase.auth.signUp({ - email: registerDto.email, - password: generate({ length: 16, numbers: true, symbols: true }), - }); - - if (!user) { - throw error ?? new Error(`Unknown error ocurred when signing up user with email ${registerDto.email}`); - } - - logger.debug(`User with email ${registerDto.email} registered`, { id: user.id }); - - return user.id; -}; - export const getUsersByRole = async (role: Role) => { logger.debug(`Fetching users with the ${role} role`); const { data, error } = await supabase.from(USERS_TABLE_NAME).select().eq('role', role); - if (!data) { + if (!data || error) { throw new Error(error ? error.message : `Unknown error ocurred when getting users from the database`); } @@ -42,10 +24,14 @@ export const getUsersByRole = async (role: Role) => { return data; }; -export const insertUsers = async (users: User[]) => { +export const insertUsers = async (users: CreateUserDTO[]) => { logger.debug(`Inserting provided users to the ${USERS_TABLE_NAME} table`, users); - await supabase.from(USERS_TABLE_NAME).insert(users); + const { error } = await supabase.from(USERS_TABLE_NAME).insert(users); + + if (error) { + throw new Error(error.message); + } logger.debug(`Users inserted to the ${USERS_TABLE_NAME} table`); }; @@ -53,7 +39,11 @@ export const insertUsers = async (users: User[]) => { export const updateUserById = async (id: User['id'], data: Partial>) => { logger.debug(`Updating user with id ${id} using the provided data`, data); - await supabase.from(USERS_TABLE_NAME).update(data).match({ id }); + const { error } = await supabase.from(USERS_TABLE_NAME).update(data).match({ id }); + + if (error) { + throw new Error(error.message); + } logger.debug(`User with id ${id} updated successfully`); }; diff --git a/packages/cli/src/shared/env.ts b/packages/cli/src/shared/env.ts index a2989a3c..ee6a10f7 100644 --- a/packages/cli/src/shared/env.ts +++ b/packages/cli/src/shared/env.ts @@ -1,5 +1,5 @@ import { Expose, plainToClass } from 'class-transformer'; -import { IsInt, IsNotEmpty, IsString, validateOrReject } from 'class-validator'; +import { IsNotEmpty, IsString, validateOrReject } from 'class-validator'; import dotenv from 'dotenv'; import { createLogger } from './logger'; @@ -23,26 +23,6 @@ class EnvVariables { @IsString() @IsNotEmpty() PROCESS_ST_CHECKLIST_URL: string; - - @Expose() - @IsInt() - // @IsNotEmpty() - NODEMAILER_PORT: number; - - @Expose() - @IsString() - // @IsNotEmpty() - NODEMAILER_HOST: string; - - @Expose() - @IsString() - // @IsNotEmpty() - NODEMAILER_USER: string; - - @Expose() - @IsString() - // @IsNotEmpty() - NODEMAILER_PASSWORD: string; } export const env = plainToClass(EnvVariables, process.env, { diff --git a/packages/cli/src/shared/files.ts b/packages/cli/src/shared/files.ts new file mode 100644 index 00000000..41fc90f1 --- /dev/null +++ b/packages/cli/src/shared/files.ts @@ -0,0 +1,43 @@ +import { existsSync } from 'fs'; +import { mkdir, writeFile } from 'fs/promises'; + +import { createLogger } from './logger'; + +const logger = createLogger('Files Utils'); + +const createOutputDir = async () => { + if (!existsSync('output')) { + await mkdir('output'); + } +}; + +export const createPasswordsList = () => { + const passwords: Record = {}; + + const add = ({ email, password }: { email: string; password: string }) => { + logger.debug(`Adding password for email ${email}`); + passwords[email] = password; + }; + + const save = async () => { + await createOutputDir(); + + const path = 'output/passwords.json'; + + logger.debug(`Saving passwords object to ${path}`); + await writeFile(path, JSON.stringify(passwords), 'utf-8'); + logger.debug('Passwords saved successfully'); + }; + + return { add, save }; +}; + +export const createWelcomeCsv = async (csv: string) => { + await createOutputDir(); + + const path = 'output/welcome.csv'; + + logger.debug(`Saving welcome CSV to ${path}`); + await writeFile(path, csv, 'utf-8'); + logger.debug('Welcome CSV saved successfully'); +}; diff --git a/packages/cli/src/shared/models.ts b/packages/cli/src/shared/models.ts index d76ec5e2..746d6ef6 100644 --- a/packages/cli/src/shared/models.ts +++ b/packages/cli/src/shared/models.ts @@ -1,6 +1,6 @@ /* eslint-disable max-classes-per-file */ import { Expose, Transform } from 'class-transformer'; -import { IsEmail, IsIn, IsNotEmpty, IsOptional, IsString, IsUrl } from 'class-validator'; +import { IsEmail, IsIn, IsNotEmpty, IsNumber, IsOptional, IsString, IsUrl } from 'class-validator'; export const userRoles = { participant: 'participant', @@ -24,22 +24,35 @@ export class ParticipantCsvRow { @Expose() @IsEmail() @IsNotEmpty() - @Transform(({ value }) => value.toLowerCase()) + @Transform(({ value }) => (typeof value === 'string' ? value.trim().toLowerCase() : value)) email: string; } -export class RegisterDTO { +export class CreateUserDTO { @Expose() @IsEmail() @IsNotEmpty() email: string; -} -export class User { @Expose() @IsString() @IsNotEmpty() - id: string; + firstName: string; + + @Expose() + @IsString() + @IsNotEmpty() + lastName: string; + + @Expose() + @IsIn(Object.values(userRoles)) + role: Role; +} + +export class User { + @Expose() + @IsNumber() + id: number; @Expose() @IsEmail() @@ -49,7 +62,12 @@ export class User { @Expose() @IsString() @IsNotEmpty() - name: string; + firstName: string; + + @Expose() + @IsString() + @IsNotEmpty() + lastName: string; @Expose() @IsUrl() @@ -60,3 +78,20 @@ export class User { @IsIn(Object.values(userRoles)) role: Role; } + +export class WelcomeCsvRow { + @Expose() + @IsString() + @IsNotEmpty() + firstName: string; + + @Expose() + @IsEmail() + @IsNotEmpty() + email: string; + + @Expose() + @IsUrl() + @IsNotEmpty() + checklist: string; +} diff --git a/packages/cli/src/shared/object.ts b/packages/cli/src/shared/object.ts index 70b909f6..710ad03a 100644 --- a/packages/cli/src/shared/object.ts +++ b/packages/cli/src/shared/object.ts @@ -1,15 +1,15 @@ import { ClassConstructor, plainToClass } from 'class-transformer'; import { validateOrReject } from 'class-validator'; -import type { AnyObject, UnknownObject } from '@coderscamp/shared/types'; +import type { AnyObject } from '@coderscamp/shared/types'; import { createLogger } from './logger'; const logger = createLogger('Object utils'); -export const transformToMatchClass = +export const transformAndValidate = (cls: ClassConstructor) => - async (obj: Obj): Promise => { + async (obj: Obj): Promise => { logger.debug(`Transforming object to match the ${cls.name} class`, obj); const result = plainToClass(cls, obj, { excludeExtraneousValues: true, enableImplicitConversion: true }); @@ -22,3 +22,31 @@ export const transformToMatchClass = return result; }; + +export const transformToMatchClass = + (cls: ClassConstructor) => + (obj: Obj): ClassInstance => { + logger.debug(`Transforming object to match the ${cls.name} class`, obj); + + const result = plainToClass(cls, obj, { excludeExtraneousValues: true, enableImplicitConversion: true }); + + logger.debug(`Object successfully transformed to match the ${cls.name} class`, result); + + return result; + }; + +export const filterInvalid = + (cls: ClassConstructor) => + async (obj: Obj): Promise => { + try { + await transformAndValidate(cls)(obj); + + logger.debug(`Item matches the ${cls.name} class`, obj); + + return true; + } catch (ex) { + logger.debug(`Item has been filtered as it doesn't match the ${cls.name} class`, obj); + + return false; + } + }; diff --git a/yarn.lock b/yarn.lock index e785413f..5dac99d6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2914,12 +2914,13 @@ __metadata: resolution: "@coderscamp/cli@workspace:packages/cli" dependencies: "@supabase/supabase-js": 1.24.0 + "@types/json2csv": 5.0.3 class-transformer: 0.4.0 class-validator: 0.13.1 commander: 8.2.0 csv-parse: 4.16.3 dotenv: 10.0.0 - generate-password: 1.6.1 + json2csv: 5.0.6 puppeteer: 10.4.0 reflect-metadata: 0.1.13 winston: 3.3.3 @@ -7197,6 +7198,15 @@ __metadata: languageName: node linkType: hard +"@types/json2csv@npm:5.0.3": + version: 5.0.3 + resolution: "@types/json2csv@npm:5.0.3" + dependencies: + "@types/node": "*" + checksum: 7a9faf4a7ec0fb488cf953ba18aea1e48fb586476e7aa9d8c6d03860b729a508cfe836170c90f3ba3656d9621df95a264bf0452f69771e8df06d1ce221980533 + languageName: node + linkType: hard + "@types/json5@npm:^0.0.29": version: 0.0.29 resolution: "@types/json5@npm:0.0.29" @@ -11366,7 +11376,7 @@ __metadata: languageName: node linkType: hard -"commander@npm:^6.2.0, commander@npm:^6.2.1": +"commander@npm:^6.1.0, commander@npm:^6.2.0, commander@npm:^6.2.1": version: 6.2.1 resolution: "commander@npm:6.2.1" checksum: d7090410c0de6bc5c67d3ca41c41760d6d268f3c799e530aafb73b7437d1826bbf0d2a3edac33f8b57cc9887b4a986dce307fa5557e109be40eadb7c43b21742 @@ -15103,13 +15113,6 @@ fsevents@^1.2.7: languageName: node linkType: hard -"generate-password@npm:1.6.1": - version: 1.6.1 - resolution: "generate-password@npm:1.6.1" - checksum: aa01fd835568436158e045a368546d275f7bb647d3e461091820fa7b53efb92285fcaca5dbd919adb8ab9f5e4064cbbfa0da697ef9c226eeb67487e6b68e8fc0 - languageName: node - linkType: hard - "gensync@npm:^1.0.0-beta.1, gensync@npm:^1.0.0-beta.2": version: 1.0.0-beta.2 resolution: "gensync@npm:1.0.0-beta.2" @@ -18187,6 +18190,19 @@ fsevents@^1.2.7: languageName: node linkType: hard +"json2csv@npm:5.0.6": + version: 5.0.6 + resolution: "json2csv@npm:5.0.6" + dependencies: + commander: ^6.1.0 + jsonparse: ^1.3.1 + lodash.get: ^4.4.2 + bin: + json2csv: bin/json2csv.js + checksum: 82cf796dbaeade644303295262ac9f554cf3924ec00b263bb094028b80f0959d89a1eaa99b3634de022916fd3278adb82ec478c57fc176827a2569f6cef1db31 + languageName: node + linkType: hard + "json3@npm:^3.3.2, json3@npm:^3.3.3": version: 3.3.3 resolution: "json3@npm:3.3.3" @@ -18260,6 +18276,13 @@ fsevents@^1.2.7: languageName: node linkType: hard +"jsonparse@npm:^1.3.1": + version: 1.3.1 + resolution: "jsonparse@npm:1.3.1" + checksum: 6514a7be4674ebf407afca0eda3ba284b69b07f9958a8d3113ef1005f7ec610860c312be067e450c569aab8b89635e332cee3696789c750692bb60daba627f4d + languageName: node + linkType: hard + "jsonwebtoken@npm:8.5.1, jsonwebtoken@npm:^8.2.0": version: 8.5.1 resolution: "jsonwebtoken@npm:8.5.1" @@ -18684,6 +18707,13 @@ fsevents@^1.2.7: languageName: node linkType: hard +"lodash.get@npm:^4.4.2": + version: 4.4.2 + resolution: "lodash.get@npm:4.4.2" + checksum: e403047ddb03181c9d0e92df9556570e2b67e0f0a930fcbbbd779370972368f5568e914f913e93f3b08f6d492abc71e14d4e9b7a18916c31fa04bd2306efe545 + languageName: node + linkType: hard + "lodash.includes@npm:^4.3.0": version: 4.3.0 resolution: "lodash.includes@npm:4.3.0"