Some basics

.gitignore

@@ -0,0 +1 @@
+/dist/

LICENSE.md

@@ -0,0 +1,24 @@
+# The MIT License (MIT)
+
+Copyright © `2019` Benjie Gillam
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the “Software”), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -22,6 +22,7 @@ Opinionated SQL-powered migration tool for PostgreSQL.
 - Migrations should automatically be wrapped in transactions
 - Migrations should not pollute PostgreSQL global settings (e.g. use `SET LOCAL` rather than `SET`)
 - Migrations that require execution outside of a transaction (e.g. to enable augmenting non-DDL-safe things, such as `ENUM`s in PostgreSQL) should be explicitly marked
+- Loading all your migrations at once into memory should not exhaust Node's memory 😉
 
 ## Setup
 
@@ -41,12 +42,14 @@ files for running graphile-migrate.
 commands in order to import the existing database as if it were the first
 migration.
 
-`graphile-migrate watch` will watch the new migration file,
-re-running it's SQL on any change. This file should be idempotent (this is
-your responsibility); i.e. it should be able to be ran multiple times and
-have the same result. Further, they should use `CASCADE` so that if other
-migrations are worked on in parallel no additional `rollback` step is
-required. Examples of idempotent operations:
+`graphile-migrate migrate` will run any un-executed committed migrations.
+
+`graphile-migrate watch` will run any un-executed committed migration and
+then watch the new migration file, re-running it's SQL on any change. This
+file should be idempotent (this is your responsibility); i.e. it should be
+able to be ran multiple times and have the same result. Further, they should
+use `CASCADE` so that if other migrations are worked on in parallel no
+additional `rollback` step is required. Examples of idempotent operations:
 
 ```
 -- Create a schema

package.json

@@ -2,9 +2,17 @@
   "name": "graphile-migrate",
   "version": "0.0.0",
   "description": "Opinionated SQL-powered migration tool for PostgreSQL",
-  "main": "index.js",
+  "main": "dist/index.js",
   "scripts": {
-    "test": "jest"
+    "prepack": "tsc && chmod +x dist/cli.js",
+    "watch": "tsc --watch"
+  },
+  "bin": {
+    "graphile-migrate": "./dist/cli.js"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+ssh://git@github.com/graphile/migrate.git"
   },
   "keywords": [
     "postgresql",
@@ -18,5 +26,19 @@
     "functions"
   ],
   "author": "Benjie Gillam <code@benjiegillam.com>",
-  "license": "MIT"
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/graphile/migrate/issues"
+  },
+  "homepage": "https://github.com/graphile/migrate#readme",
+  "dependencies": {
+    "@types/pg": "^7.4.11",
+    "pg": ">=6.5 <8"
+  },
+  "devDependencies": {
+    "typescript": "^3.3.1"
+  },
+  "files": [
+    "dist"
+  ]
 }

src/cli.ts

@@ -0,0 +1,36 @@
+#!/usr/bin/env node
+import * as fs from "fs";
+import { migrate } from "./index";
+
+function getSettings() {
+  let data;
+  try {
+    data = fs.readFileSync(`${process.cwd()}/.gmrc`, "utf8");
+  } catch (e) {
+    throw new Error(
+      "No .gmrc file found; please run `graphile-migrate init` first."
+    );
+  }
+  try {
+    return JSON.parse(data);
+  } catch (e) {
+    throw new Error("Failed to parse .gmrc file: " + e.message);
+  }
+}
+
+async function main() {
+  const argv = process.argv.slice(2);
+  const [cmd] = argv;
+  if (cmd === "migrate") {
+    await migrate(getSettings().connectionString);
+  } else {
+    console.error(`Command '${cmd || ""}' not understood`);
+    process.exit(1);
+  }
+}
+
+main().catch(e => {
+  // tslint:disable-next-line no-console
+  console.error(e);
+  process.exit(1);
+});

src/index.ts

@@ -0,0 +1,230 @@
+import * as pg from "pg";
+import * as fs from "fs";
+import * as path from "path";
+import * as crypto from "crypto";
+import { promisify } from "util";
+
+const calculateHash = (str: string) =>
+  crypto
+    .createHash("sha1")
+    .update(str)
+    .digest("hex");
+
+// NEVER CHANGE THESE!
+const PREVIOUS = "--! Previous: ";
+const HASH = "--! Hash: ";
+
+const fsp = {
+  readFile: promisify(fs.readFile),
+  stat: promisify(fs.stat),
+  readdir: promisify(fs.readdir),
+  mkdir: promisify(fs.mkdir),
+};
+
+interface Migration {
+  filename: string;
+  hash: string;
+  previousHash: string | null;
+}
+
+interface DbMigration extends Migration {
+  date: Date;
+}
+
+interface FileMigration extends Migration {
+  body: string;
+  fullPath: string;
+  previous: FileMigration | null;
+}
+
+async function migrateMigrationSchema(pgClient: pg.PoolClient) {
+  await pgClient.query(`
+    create schema if not exists graphile_migrate;
+
+    create table if not exists graphile_migrate.migrations (
+      hash text primary key,
+      previous_hash text references graphile_migrate.migrations,
+      filename text not null,
+      date timestamptz not null default now()
+    );
+  `);
+}
+
+async function getLastMigration(
+  pgClient: pg.PoolClient
+): Promise<DbMigration | null> {
+  await migrateMigrationSchema(pgClient);
+  const {
+    rows: [row],
+  } = await pgClient.query(
+    `select filename, previous_hash as "previousHash", hash, date from graphile_migrate.migrations order by filename desc limit 1`
+  );
+  return (row as DbMigration) || null;
+}
+
+async function getAllMigrations(): Promise<Array<FileMigration>> {
+  const migrationsFolder = `${process.cwd()}/migrations/committed`;
+  const stat = await fsp.stat(migrationsFolder);
+  if (stat) {
+    if (!stat.isDirectory()) {
+      throw new Error(`${migrationsFolder} is not a directory`);
+    }
+  } else {
+    await fsp.mkdir(path.dirname(migrationsFolder));
+    await fsp.mkdir(migrationsFolder);
+  }
+  const files = await fsp.readdir(migrationsFolder);
+  const isMigration = (filename: string) => filename.match(/^[0-9]{6}_.*\.sql/);
+  const migrations: Array<FileMigration> = await Promise.all(
+    files.filter(isMigration).map(
+      async (filename): Promise<FileMigration> => {
+        const fullPath = `${migrationsFolder}/${filename}`;
+        const contents = await fsp.readFile(fullPath, "utf8");
+        const i = contents.indexOf("\n");
+        const firstLine = contents.substring(0, i);
+        if (!firstLine.startsWith(PREVIOUS)) {
+          throw new Error(
+            "Invalid committed migration - no 'previous' comment"
+          );
+        }
+        const previousHash = firstLine.substring(PREVIOUS.length) || null;
+        const j = contents.indexOf("\n", i + 1);
+        const secondLine = contents.substring(i + 1, j);
+        if (!secondLine.startsWith(HASH)) {
+          throw new Error("Invalid committed migration - no 'hash' comment");
+        }
+        const hash = secondLine.substring(HASH.length);
+        const body = contents.substring(j + 1).trim();
+        return {
+          filename,
+          fullPath,
+          hash,
+          previousHash,
+          body,
+          previous: null,
+        };
+      }
+    )
+  );
+  migrations.sort((a, b) => a.filename.localeCompare(b.filename));
+  // Validate and link
+  let previous = null;
+  for (const migration of migrations) {
+    if (!previous) {
+      if (migration.previousHash !== null) {
+        throw new Error(
+          `Migration '${
+            migration.filename
+          }' expected a previous migration, but no correctly ordered previous migration was found`
+        );
+      }
+    } else {
+      if (migration.previousHash !== previous.hash) {
+        throw new Error(
+          `Previous migration with hash '${previous.hash}' doesn't match '${
+            migration.filename
+          }''s expected previous hash ${migration.previousHash}`
+        );
+      }
+    }
+    migration.previous = previous;
+    previous = migration;
+  }
+  return migrations;
+}
+
+async function getMigrationsAfter(
+  previousMigration: Migration | null
+): Promise<Array<FileMigration>> {
+  const allMigrations = await getAllMigrations();
+  return allMigrations.filter(
+    m => !previousMigration || m.filename > previousMigration.filename
+  );
+}
+
+async function runStringMigration(
+  pgClient: pg.PoolClient,
+  body: string,
+  committedMigration?: FileMigration
+) {
+  const i = body.indexOf("\n");
+  const firstLine = body.substring(0, i);
+  const transaction = !firstLine.match(/^--!\s*no-transaction\b/);
+  if (transaction) {
+    await pgClient.query("begin");
+  }
+  try {
+    await pgClient.query(body);
+  } catch (e) {
+    // tslint:disable-next-line no-console
+    console.error(`Error occurred whilst processing migration: ${e.message}`);
+    process.exit(1);
+  }
+  if (committedMigration) {
+    const { hash, previousHash, filename } = committedMigration;
+    await pgClient.query({
+      name: "migration-insert",
+      text:
+        "insert into graphile_migrate.migrations(hash, previous_hash, filename) values ($1, $2, $3)",
+      values: [hash, previousHash, filename],
+    });
+  }
+  if (transaction) {
+    await pgClient.query("commit");
+  }
+}
+
+async function runCommittedMigration(
+  pgClient: pg.PoolClient,
+  committedMigration: FileMigration
+) {
+  const { hash, filename, body } = committedMigration;
+  // Check the hash
+  const newHash = calculateHash(body);
+  if (newHash !== hash) {
+    throw new Error(
+      `Hash for ${filename} does not match - ${newHash} !== ${hash}; has the file been tampered with?`
+    );
+  }
+  // tslint:disable-next-line no-console
+  console.log(`graphile-migrate: Running migration '${filename}'`);
+  await runStringMigration(pgClient, body, committedMigration);
+}
+
+export async function migrate(connectionString: string) {
+  await withClient(connectionString, async pgClient => {
+    const lastMigration = await getLastMigration(pgClient);
+    const remainingMigrations = await getMigrationsAfter(lastMigration);
+    if (remainingMigrations.length === 0) {
+      // tslint:disable-next-line no-console
+      console.log("graphile-migrate: Up to date");
+    } else {
+      // Run migrations in series
+      for (const migration of remainingMigrations) {
+        await runCommittedMigration(pgClient, migration);
+      }
+    }
+  });
+}
+
+async function withClient(
+  connectionString: string,
+  callback: (pgClient: pg.PoolClient) => Promise<void>
+) {
+  const pgPool = new pg.Pool({ connectionString });
+  pgPool.on("error", (err: Error) => {
+    // tslint:disable-next-line no-console
+    console.error("An error occurred in the PgPool", err);
+    process.exit(1);
+  });
+  try {
+    const pgClient = await pgPool.connect();
+    try {
+      await callback(pgClient);
+    } finally {
+      await pgClient.release();
+    }
+  } finally {
+    await pgPool.end();
+  }
+}

tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "declarationDir": "./dist",
+    "outDir": "./dist"
+  },
+  "include": ["src/**/*"]
+}