diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..178135c --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/dist/ diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..d94b5bb --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,24 @@ +# The MIT License (MIT) + +Copyright © `2019` Benjie Gillam + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the “Software”), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index 0adb98d..172656b 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Opinionated SQL-powered migration tool for PostgreSQL. - Migrations should automatically be wrapped in transactions - Migrations should not pollute PostgreSQL global settings (e.g. use `SET LOCAL` rather than `SET`) - Migrations that require execution outside of a transaction (e.g. to enable augmenting non-DDL-safe things, such as `ENUM`s in PostgreSQL) should be explicitly marked +- Loading all your migrations at once into memory should not exhaust Node's memory 😉 ## Setup @@ -41,12 +42,14 @@ files for running graphile-migrate. commands in order to import the existing database as if it were the first migration. -`graphile-migrate watch` will watch the new migration file, -re-running it's SQL on any change. This file should be idempotent (this is -your responsibility); i.e. it should be able to be ran multiple times and -have the same result. Further, they should use `CASCADE` so that if other -migrations are worked on in parallel no additional `rollback` step is -required. Examples of idempotent operations: +`graphile-migrate migrate` will run any un-executed committed migrations. + +`graphile-migrate watch` will run any un-executed committed migration and +then watch the new migration file, re-running it's SQL on any change. This +file should be idempotent (this is your responsibility); i.e. it should be +able to be ran multiple times and have the same result. Further, they should +use `CASCADE` so that if other migrations are worked on in parallel no +additional `rollback` step is required. Examples of idempotent operations: ``` -- Create a schema diff --git a/package.json b/package.json index 3e43737..b52f6d7 100644 --- a/package.json +++ b/package.json @@ -2,9 +2,17 @@ "name": "graphile-migrate", "version": "0.0.0", "description": "Opinionated SQL-powered migration tool for PostgreSQL", - "main": "index.js", + "main": "dist/index.js", "scripts": { - "test": "jest" + "prepack": "tsc && chmod +x dist/cli.js", + "watch": "tsc --watch" + }, + "bin": { + "graphile-migrate": "./dist/cli.js" + }, + "repository": { + "type": "git", + "url": "git+ssh://git@github.com/graphile/migrate.git" }, "keywords": [ "postgresql", @@ -18,5 +26,19 @@ "functions" ], "author": "Benjie Gillam ", - "license": "MIT" + "license": "MIT", + "bugs": { + "url": "https://github.com/graphile/migrate/issues" + }, + "homepage": "https://github.com/graphile/migrate#readme", + "dependencies": { + "@types/pg": "^7.4.11", + "pg": ">=6.5 <8" + }, + "devDependencies": { + "typescript": "^3.3.1" + }, + "files": [ + "dist" + ] } diff --git a/src/cli.ts b/src/cli.ts new file mode 100644 index 0000000..b9235f3 --- /dev/null +++ b/src/cli.ts @@ -0,0 +1,36 @@ +#!/usr/bin/env node +import * as fs from "fs"; +import { migrate } from "./index"; + +function getSettings() { + let data; + try { + data = fs.readFileSync(`${process.cwd()}/.gmrc`, "utf8"); + } catch (e) { + throw new Error( + "No .gmrc file found; please run `graphile-migrate init` first." + ); + } + try { + return JSON.parse(data); + } catch (e) { + throw new Error("Failed to parse .gmrc file: " + e.message); + } +} + +async function main() { + const argv = process.argv.slice(2); + const [cmd] = argv; + if (cmd === "migrate") { + await migrate(getSettings().connectionString); + } else { + console.error(`Command '${cmd || ""}' not understood`); + process.exit(1); + } +} + +main().catch(e => { + // tslint:disable-next-line no-console + console.error(e); + process.exit(1); +}); diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..cc02487 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,230 @@ +import * as pg from "pg"; +import * as fs from "fs"; +import * as path from "path"; +import * as crypto from "crypto"; +import { promisify } from "util"; + +const calculateHash = (str: string) => + crypto + .createHash("sha1") + .update(str) + .digest("hex"); + +// NEVER CHANGE THESE! +const PREVIOUS = "--! Previous: "; +const HASH = "--! Hash: "; + +const fsp = { + readFile: promisify(fs.readFile), + stat: promisify(fs.stat), + readdir: promisify(fs.readdir), + mkdir: promisify(fs.mkdir), +}; + +interface Migration { + filename: string; + hash: string; + previousHash: string | null; +} + +interface DbMigration extends Migration { + date: Date; +} + +interface FileMigration extends Migration { + body: string; + fullPath: string; + previous: FileMigration | null; +} + +async function migrateMigrationSchema(pgClient: pg.PoolClient) { + await pgClient.query(` + create schema if not exists graphile_migrate; + + create table if not exists graphile_migrate.migrations ( + hash text primary key, + previous_hash text references graphile_migrate.migrations, + filename text not null, + date timestamptz not null default now() + ); + `); +} + +async function getLastMigration( + pgClient: pg.PoolClient +): Promise { + await migrateMigrationSchema(pgClient); + const { + rows: [row], + } = await pgClient.query( + `select filename, previous_hash as "previousHash", hash, date from graphile_migrate.migrations order by filename desc limit 1` + ); + return (row as DbMigration) || null; +} + +async function getAllMigrations(): Promise> { + const migrationsFolder = `${process.cwd()}/migrations/committed`; + const stat = await fsp.stat(migrationsFolder); + if (stat) { + if (!stat.isDirectory()) { + throw new Error(`${migrationsFolder} is not a directory`); + } + } else { + await fsp.mkdir(path.dirname(migrationsFolder)); + await fsp.mkdir(migrationsFolder); + } + const files = await fsp.readdir(migrationsFolder); + const isMigration = (filename: string) => filename.match(/^[0-9]{6}_.*\.sql/); + const migrations: Array = await Promise.all( + files.filter(isMigration).map( + async (filename): Promise => { + const fullPath = `${migrationsFolder}/${filename}`; + const contents = await fsp.readFile(fullPath, "utf8"); + const i = contents.indexOf("\n"); + const firstLine = contents.substring(0, i); + if (!firstLine.startsWith(PREVIOUS)) { + throw new Error( + "Invalid committed migration - no 'previous' comment" + ); + } + const previousHash = firstLine.substring(PREVIOUS.length) || null; + const j = contents.indexOf("\n", i + 1); + const secondLine = contents.substring(i + 1, j); + if (!secondLine.startsWith(HASH)) { + throw new Error("Invalid committed migration - no 'hash' comment"); + } + const hash = secondLine.substring(HASH.length); + const body = contents.substring(j + 1).trim(); + return { + filename, + fullPath, + hash, + previousHash, + body, + previous: null, + }; + } + ) + ); + migrations.sort((a, b) => a.filename.localeCompare(b.filename)); + // Validate and link + let previous = null; + for (const migration of migrations) { + if (!previous) { + if (migration.previousHash !== null) { + throw new Error( + `Migration '${ + migration.filename + }' expected a previous migration, but no correctly ordered previous migration was found` + ); + } + } else { + if (migration.previousHash !== previous.hash) { + throw new Error( + `Previous migration with hash '${previous.hash}' doesn't match '${ + migration.filename + }''s expected previous hash ${migration.previousHash}` + ); + } + } + migration.previous = previous; + previous = migration; + } + return migrations; +} + +async function getMigrationsAfter( + previousMigration: Migration | null +): Promise> { + const allMigrations = await getAllMigrations(); + return allMigrations.filter( + m => !previousMigration || m.filename > previousMigration.filename + ); +} + +async function runStringMigration( + pgClient: pg.PoolClient, + body: string, + committedMigration?: FileMigration +) { + const i = body.indexOf("\n"); + const firstLine = body.substring(0, i); + const transaction = !firstLine.match(/^--!\s*no-transaction\b/); + if (transaction) { + await pgClient.query("begin"); + } + try { + await pgClient.query(body); + } catch (e) { + // tslint:disable-next-line no-console + console.error(`Error occurred whilst processing migration: ${e.message}`); + process.exit(1); + } + if (committedMigration) { + const { hash, previousHash, filename } = committedMigration; + await pgClient.query({ + name: "migration-insert", + text: + "insert into graphile_migrate.migrations(hash, previous_hash, filename) values ($1, $2, $3)", + values: [hash, previousHash, filename], + }); + } + if (transaction) { + await pgClient.query("commit"); + } +} + +async function runCommittedMigration( + pgClient: pg.PoolClient, + committedMigration: FileMigration +) { + const { hash, filename, body } = committedMigration; + // Check the hash + const newHash = calculateHash(body); + if (newHash !== hash) { + throw new Error( + `Hash for ${filename} does not match - ${newHash} !== ${hash}; has the file been tampered with?` + ); + } + // tslint:disable-next-line no-console + console.log(`graphile-migrate: Running migration '${filename}'`); + await runStringMigration(pgClient, body, committedMigration); +} + +export async function migrate(connectionString: string) { + await withClient(connectionString, async pgClient => { + const lastMigration = await getLastMigration(pgClient); + const remainingMigrations = await getMigrationsAfter(lastMigration); + if (remainingMigrations.length === 0) { + // tslint:disable-next-line no-console + console.log("graphile-migrate: Up to date"); + } else { + // Run migrations in series + for (const migration of remainingMigrations) { + await runCommittedMigration(pgClient, migration); + } + } + }); +} + +async function withClient( + connectionString: string, + callback: (pgClient: pg.PoolClient) => Promise +) { + const pgPool = new pg.Pool({ connectionString }); + pgPool.on("error", (err: Error) => { + // tslint:disable-next-line no-console + console.error("An error occurred in the PgPool", err); + process.exit(1); + }); + try { + const pgClient = await pgPool.connect(); + try { + await callback(pgClient); + } finally { + await pgClient.release(); + } + } finally { + await pgPool.end(); + } +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..d26c39d --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "rootDir": "src", + "declarationDir": "./dist", + "outDir": "./dist" + }, + "include": ["src/**/*"] +}