From a7334cfa8449de735b03b982ae7dffbe768d18b9 Mon Sep 17 00:00:00 2001 From: Dominic Saadi Date: Tue, 23 May 2023 20:14:41 -0700 Subject: [PATCH] fix(data-migrate): provide option to use db in dist instead of src (#8375) * separate out up, install handlers * remove needless async * add missing @ * fix import * use values not entries * fix data structure * add back early return * rename to camel case * feat: support `--dist-path` flag * follow up fixes * apply suggestions from review --- .../{data-migrate.js => dataMigrate.js} | 13 +- .../cli/src/commands/dataMigrate/install.js | 91 +------ .../commands/dataMigrate/installHandler.js | 79 ++++++ packages/cli/src/commands/dataMigrate/up.js | 196 ++------------- .../cli/src/commands/dataMigrate/upHandler.js | 232 ++++++++++++++++++ .../generate/dataMigration/dataMigration.js | 3 +- packages/cli/src/index.js | 2 +- 7 files changed, 355 insertions(+), 261 deletions(-) rename packages/cli/src/commands/{data-migrate.js => dataMigrate.js} (65%) create mode 100644 packages/cli/src/commands/dataMigrate/installHandler.js create mode 100644 packages/cli/src/commands/dataMigrate/upHandler.js diff --git a/packages/cli/src/commands/data-migrate.js b/packages/cli/src/commands/dataMigrate.js similarity index 65% rename from packages/cli/src/commands/data-migrate.js rename to packages/cli/src/commands/dataMigrate.js index 6d35ae664c47..d52da10e0a76 100644 --- a/packages/cli/src/commands/data-migrate.js +++ b/packages/cli/src/commands/dataMigrate.js @@ -1,15 +1,20 @@ +import terminalLink from 'terminal-link' + +import * as installCommand from './dataMigrate/install' +import * as upCommand from './dataMigrate/up' + export const command = 'data-migrate ' export const aliases = ['dm', 'dataMigrate'] export const description = 'Migrate the data in your database' -import terminalLink from 'terminal-link' -export const builder = (yargs) => +export function builder(yargs) { yargs - .commandDir('./dataMigrate') - .demandCommand() + .command(installCommand) + .command(upCommand) .epilogue( `Also see the ${terminalLink( 'Redwood CLI Reference', 'https://redwoodjs.com/docs/cli-commands#datamigrate' )}` ) +} diff --git a/packages/cli/src/commands/dataMigrate/install.js b/packages/cli/src/commands/dataMigrate/install.js index 87845786e8d3..215cafecfa12 100644 --- a/packages/cli/src/commands/dataMigrate/install.js +++ b/packages/cli/src/commands/dataMigrate/install.js @@ -1,61 +1,9 @@ -import path from 'path' - -import execa from 'execa' -import fs from 'fs-extra' -import { Listr } from 'listr2' import terminalLink from 'terminal-link' -import { errorTelemetry } from '@redwoodjs/telemetry' - -import { getPaths } from '../../lib' -import c from '../../lib/colors' - -const MODEL = `model RW_DataMigration { - version String @id - name String - startedAt DateTime - finishedAt DateTime -}` - -const POST_INSTALL_INSTRUCTIONS = `${c.warning( - "Don't forget to apply your migration when ready:" -)} - - ${c.bold('yarn rw prisma migrate dev')} -` - -// Creates dataMigrations directory -const createPath = () => { - return fs.outputFileSync( - path.join(getPaths().api.dataMigrations, '.keep'), - '' - ) -} - -// Appends RW_DataMigration model to schema.prisma -const appendModel = () => { - const schemaPath = getPaths().api.dbSchema - const schema = fs.readFileSync(schemaPath).toString() - const newSchema = `${schema}\n${MODEL}\n` - - return fs.writeFileSync(schemaPath, newSchema) -} - -// Create a new migration -const save = async () => { - return await execa( - 'yarn rw', - ['prisma migrate dev', '--name create_data_migrations', '--create-only'], - { - cwd: getPaths().api.base, - shell: true, - } - ) -} - export const command = 'install' export const description = 'Add the RW_DataMigration model to your schema' -export const builder = (yargs) => { + +export function builder(yargs) { yargs.epilogue( `Also see the ${terminalLink( 'Redwood CLI Reference', @@ -64,36 +12,7 @@ export const builder = (yargs) => { ) } -export const handler = async () => { - const tasks = new Listr( - [ - { - title: `Creating dataMigrations directory...`, - task: createPath, - }, - { - title: 'Adding RW_DataMigration model to schema.prisma...', - task: await appendModel, - }, - { - title: 'Create db migration...', - task: await save, - }, - { - title: 'One more thing...', - task: (_ctx, task) => { - task.title = `Next steps:\n ${POST_INSTALL_INSTRUCTIONS}` - }, - }, - ], - { rendererOptions: { collapseSubtasks: false }, exitOnError: true } - ) - - try { - await tasks.run() - } catch (e) { - errorTelemetry(process.argv, e.message) - console.error(c.error(e.message)) - process.exit(e?.exitCode || 1) - } +export async function handler(options) { + const { handler } = await import('./installHandler') + return handler(options) } diff --git a/packages/cli/src/commands/dataMigrate/installHandler.js b/packages/cli/src/commands/dataMigrate/installHandler.js new file mode 100644 index 000000000000..334e05686b51 --- /dev/null +++ b/packages/cli/src/commands/dataMigrate/installHandler.js @@ -0,0 +1,79 @@ +import path from 'path' + +import execa from 'execa' +import fs from 'fs-extra' +import { Listr } from 'listr2' + +import { errorTelemetry } from '@redwoodjs/telemetry' + +import { getPaths } from '../../lib' +import c from '../../lib/colors' + +const redwoodProjectPaths = getPaths() + +export async function handler() { + const tasks = new Listr( + [ + { + title: `Creating the dataMigrations directory...`, + task() { + fs.outputFileSync( + path.join(getPaths().api.dataMigrations, '.keep'), + '' + ) + }, + }, + { + title: 'Adding the RW_DataMigration model to schema.prisma...', + task() { + const dbSchemaPath = redwoodProjectPaths.api.dbSchema + + const dbSchema = fs.readFileSync(dbSchemaPath, 'utf-8') + const newDbSchema = [dbSchema, RW_DATA_MIGRATION_MODEL, ''].join('\n') + + fs.writeFileSync(dbSchemaPath, newDbSchema) + }, + }, + { + title: 'Creating the database migration...', + task() { + return execa.command( + 'yarn rw prisma migrate dev --name create_data_migrations --create-only', + { + cwd: redwoodProjectPaths.api.base, + } + ).stdout + }, + }, + { + title: 'One more thing...', + task(_ctx, task) { + task.title = [ + 'Next steps:', + c.warning( + "Don't forget to apply your migration when you're ready:" + ), + c.bold('yarn rw prisma migrate dev'), + ].join('\n') + }, + }, + ], + { rendererOptions: { collapseSubtasks: false }, exitOnError: true } + ) + + try { + await tasks.run() + } catch (e) { + errorTelemetry(process.argv, e.message) + console.error(c.error(e.message)) + process.exit(e?.exitCode || 1) + } +} + +const RW_DATA_MIGRATION_MODEL = `\ +model RW_DataMigration { + version String @id + name String + startedAt DateTime + finishedAt DateTime +}` diff --git a/packages/cli/src/commands/dataMigrate/up.js b/packages/cli/src/commands/dataMigrate/up.js index fe58eb795048..bf7059cba435 100644 --- a/packages/cli/src/commands/dataMigrate/up.js +++ b/packages/cli/src/commands/dataMigrate/up.js @@ -1,179 +1,37 @@ -import fs from 'fs' -import path from 'path' - -import { Listr } from 'listr2' import terminalLink from 'terminal-link' -import { registerApiSideBabelHook } from '@redwoodjs/internal/dist/build/babel/api' -import { errorTelemetry } from '@redwoodjs/telemetry' - import { getPaths } from '../../lib' -import c from '../../lib/colors' - -// sorts migrations by date, oldest first -const sortMigrations = (migrations) => { - return migrations.sort((a, b) => { - const aVersion = parseInt(Object.keys(a)[0]) - const bVersion = parseInt(Object.keys(b)[0]) - - if (aVersion > bVersion) { - return 1 - } - if (aVersion < bVersion) { - return -1 - } - return 0 - }) -} - -const SUPPORTED_EXTENSIONS = ['.js', '.ts'] - -// Return the list of migrations that haven't run against the database yet -const getMigrations = async (db) => { - const basePath = path.join(getPaths().api.dataMigrations) - - if (!fs.existsSync(basePath)) { - return [] - } - - // gets all migrations present in the app - const files = fs - .readdirSync(basePath) - .filter((m) => SUPPORTED_EXTENSIONS.includes(path.extname(m))) - .map((m) => { - return { - [m.split('-')[0]]: path.join(basePath, m), - } - }) - - // gets all migration versions that have already run against the database - const ranMigrations = await db.rW_DataMigration.findMany({ - orderBy: { version: 'asc' }, - }) - const ranVersions = ranMigrations.map((migration) => - migration.version.toString() - ) - - const unrunMigrations = files.filter((migration) => { - return !ranVersions.includes(Object.keys(migration)[0]) - }) - - return sortMigrations(unrunMigrations) -} - -// adds data for completed migrations to the DB -const record = async (db, { version, name, startedAt, finishedAt }) => { - await db.rW_DataMigration.create({ - data: { version, name, startedAt, finishedAt }, - }) -} - -// output run status to the console -const report = (counters) => { - console.log('') - if (counters.run) { - console.info( - c.green(`${counters.run} data migration(s) completed successfully.`) - ) - } - if (counters.error) { - console.error( - c.error(`${counters.error} data migration(s) exited with errors.`) - ) - } - if (counters.skipped) { - console.warn( - c.warning( - `${counters.skipped} data migration(s) skipped due to previous error.` - ) - ) - } - console.log('') -} - -const runScript = async (db, scriptPath) => { - const script = await import(scriptPath) - const startedAt = new Date() - await script.default({ db }) - const finishedAt = new Date() - - return { startedAt, finishedAt } -} export const command = 'up' export const description = 'Run any outstanding Data Migrations against the database' -export const builder = (yargs) => { - yargs.epilogue( - `Also see the ${terminalLink( - 'Redwood CLI Reference', - 'https://redwoodjs.com/docs/cli-commands#datamigrate-up' - )}` - ) -} - -export const handler = async () => { - // Import babel settings so we can write es6 scripts - registerApiSideBabelHook() - - const { db } = require(path.join(getPaths().api.lib, 'db')) - const migrations = await getMigrations(db) - - // exit immediately if there aren't any migrations to run - if (!migrations.length) { - console.info(c.green('\nNo data migrations run, already up-to-date.\n')) - process.exit(0) - } - - const counters = { run: 0, skipped: 0, error: 0 } - const migrationTasks = migrations.map((migration) => { - const version = Object.keys(migration)[0] - const migrationPath = Object.values(migration)[0] - const migrationName = path.basename(migrationPath, '.js') - - return { - title: migrationName, - skip: () => { - if (counters.error > 0) { - counters.skipped++ - return true - } - }, - task: async () => { - try { - const { startedAt, finishedAt } = await runScript(db, migrationPath) - counters.run++ - await record(db, { - version, - name: migrationName, - startedAt, - finishedAt, - }) - } catch (e) { - counters.error++ - console.error(c.error(`Error in data migration: ${e.message}`)) - } - }, - } - }) - - const tasks = new Listr(migrationTasks, { - rendererOptions: { collapseSubtasks: false }, - renderer: 'verbose', - }) +/** + * @param {import('@types/yargs').Argv} yargs + */ +export function builder(yargs) { + yargs + .option('import-db-client-from-dist', { + type: 'boolean', + alias: ['db-from-dist'], + description: 'Import the db client from dist', + default: false, + }) + .option('dist-path', { + type: 'string', + alias: 'd', + description: 'Path to the api dist directory', + default: getPaths().api.dist, + }) + .epilogue( + `Also see the ${terminalLink( + 'Redwood CLI Reference', + 'https://redwoodjs.com/docs/cli-commands#datamigrate-up' + )}` + ) +} - try { - await tasks.run() - await db.$disconnect() - report(counters) - if (counters.error) { - process.exit(1) - } - } catch (e) { - await db.$disconnect() - report(counters) - errorTelemetry(process.argv, e.message) - process.exit(e?.exitCode || 1) - } +export async function handler(options) { + const { handler } = await import('./upHandler') + return handler(options) } diff --git a/packages/cli/src/commands/dataMigrate/upHandler.js b/packages/cli/src/commands/dataMigrate/upHandler.js new file mode 100644 index 000000000000..3ba5307fa6b2 --- /dev/null +++ b/packages/cli/src/commands/dataMigrate/upHandler.js @@ -0,0 +1,232 @@ +import fs from 'fs' +import path from 'path' + +import { Listr } from 'listr2' + +import { registerApiSideBabelHook } from '@redwoodjs/internal/dist/build/babel/api' +import { errorTelemetry } from '@redwoodjs/telemetry' + +import { getPaths } from '../../lib' +import c from '../../lib/colors' + +const redwoodProjectPaths = getPaths() +let requireHookRegistered = false + +/** + * @param {{ + * importDbClientFromDist: boolean + * distPath: string + * }} options + */ +export async function handler({ importDbClientFromDist, distPath }) { + let db + + if (importDbClientFromDist) { + if (!fs.existsSync(distPath)) { + console.warn( + `Can't find api dist at ${distPath}. You may need to build first: yarn rw build api` + ) + process.exitCode = 1 + return + } + + const distLibDbPath = path.join(distPath, 'lib', 'db.js') + + if (!fs.existsSync(distLibDbPath)) { + console.error( + `Can't find db.js at ${distLibDbPath}. Redwood expects the db.js file to be in the ${path.join( + distPath, + 'lib' + )} directory` + ) + process.exitCode = 1 + return + } + + db = (await import(distLibDbPath)).db + } else { + registerApiSideBabelHook() + requireHookRegistered = true + + db = (await import(path.join(redwoodProjectPaths.api.lib, 'db'))).db + } + + const pendingDataMigrations = await getPendingDataMigrations(db) + + if (!pendingDataMigrations.length) { + console.info( + c.green('\nNo pending data migrations run, already up-to-date.\n') + ) + process.exitCode = 0 + return + } + + const counters = { run: 0, skipped: 0, error: 0 } + + const dataMigrationTasks = pendingDataMigrations.map((dataMigration) => { + const dataMigrationName = path.basename(dataMigration.path, '.js') + + return { + title: dataMigrationName, + skip() { + if (counters.error > 0) { + counters.skipped++ + return true + } + }, + async task() { + if (!requireHookRegistered) { + registerApiSideBabelHook() + } + + try { + const { startedAt, finishedAt } = await runDataMigration( + db, + dataMigration.path + ) + counters.run++ + await recordDataMigration(db, { + version: dataMigration.version, + name: dataMigrationName, + startedAt, + finishedAt, + }) + } catch (e) { + counters.error++ + console.error(c.error(`Error in data migration: ${e.message}`)) + } + }, + } + }) + + const tasks = new Listr(dataMigrationTasks, { + rendererOptions: { collapseSubtasks: false }, + renderer: 'verbose', + }) + + try { + await tasks.run() + await db.$disconnect() + + console.log() + reportDataMigrations(counters) + console.log() + + if (counters.error) { + process.exitCode = 1 + } + } catch (e) { + await db.$disconnect() + + console.log() + reportDataMigrations(counters) + console.log() + + errorTelemetry(process.argv, e.message) + process.exitCode = e?.exitCode ?? 1 + } +} + +/** + * Return the list of migrations that haven't run against the database yet + */ +async function getPendingDataMigrations(db) { + const dataMigrationsPath = redwoodProjectPaths.api.dataMigrations + + if (!fs.existsSync(dataMigrationsPath)) { + return [] + } + + const dataMigrations = fs + .readdirSync(dataMigrationsPath) + // There may be a `.keep` file in the data migrations directory. + .filter((dataMigrationFileName) => + ['js', '.ts'].some((extension) => + dataMigrationFileName.endsWith(extension) + ) + ) + .map((dataMigrationFileName) => { + const [version] = dataMigrationFileName.split('-') + + return { + version, + path: path.join(dataMigrationsPath, dataMigrationFileName), + } + }) + + const ranDataMigrations = await db.rW_DataMigration.findMany({ + orderBy: { version: 'asc' }, + }) + const ranDataMigrationVersions = ranDataMigrations.map((dataMigration) => + dataMigration.version.toString() + ) + + const pendingDataMigrations = dataMigrations + .filter(({ version }) => { + return !ranDataMigrationVersions.includes(version) + }) + .sort(sortDataMigrationsByVersion) + + return pendingDataMigrations +} + +/** + * Sorts migrations by date, oldest first + */ +function sortDataMigrationsByVersion(dataMigrationA, dataMigrationB) { + const aVersion = parseInt(dataMigrationA.version) + const bVersion = parseInt(dataMigrationB.version) + + if (aVersion > bVersion) { + return 1 + } + if (aVersion < bVersion) { + return -1 + } + return 0 +} + +async function runDataMigration(db, dataMigrationPath) { + const dataMigration = await import(dataMigrationPath) + + const startedAt = new Date() + await dataMigration.default({ db }) + const finishedAt = new Date() + + return { startedAt, finishedAt } +} + +/** + * Adds data for completed migrations to the DB + */ +async function recordDataMigration( + db, + { version, name, startedAt, finishedAt } +) { + await db.rW_DataMigration.create({ + data: { version, name, startedAt, finishedAt }, + }) +} + +/** + * Output run status to the console + */ +function reportDataMigrations(counters) { + if (counters.run) { + console.info( + c.green(`${counters.run} data migration(s) completed successfully.`) + ) + } + if (counters.error) { + console.error( + c.error(`${counters.error} data migration(s) exited with errors.`) + ) + } + if (counters.skipped) { + console.warn( + c.warning( + `${counters.skipped} data migration(s) skipped due to previous error.` + ) + ) + } +} diff --git a/packages/cli/src/commands/generate/dataMigration/dataMigration.js b/packages/cli/src/commands/generate/dataMigration/dataMigration.js index 0e878727bb1d..7a22055bff45 100644 --- a/packages/cli/src/commands/generate/dataMigration/dataMigration.js +++ b/packages/cli/src/commands/generate/dataMigration/dataMigration.js @@ -35,7 +35,8 @@ export const files = ({ name, typescript }) => { } } -export const command = 'dataMigration ' +export const command = 'data-migration ' +export const aliases = ['dataMigration', 'dm'] export const description = 'Generate a data migration' export const builder = (yargs) => { yargs diff --git a/packages/cli/src/index.js b/packages/cli/src/index.js index 95c773fdc8ad..7f2b4584e92c 100644 --- a/packages/cli/src/index.js +++ b/packages/cli/src/index.js @@ -12,7 +12,7 @@ import { telemetryMiddleware } from '@redwoodjs/telemetry' import * as buildCommand from './commands/build' import * as checkCommand from './commands/check' import * as consoleCommand from './commands/console' -import * as dataMigrateCommand from './commands/data-migrate' +import * as dataMigrateCommand from './commands/dataMigrate' import * as deployCommand from './commands/deploy' import * as destroyCommand from './commands/destroy' import * as devCommand from './commands/dev'