feat: Default to SQLite Database & Better Migrations (#166)

packages/@best/api-db/README.md

@@ -2,41 +2,35 @@
 
 This is the database adapter that the frontend uses to display results. The results are stored whenever a benchmark is run.
 
-Below you can find instructions for using either Postgres or SQLite.
+Below you can find instructions for using either Postgres or SQLite. By default Best uses a local SQLite file, however we recommend using Postgres if you are running on anything other than your local machine.
 
-## Postgres
+## SQLite
+SQLite is configured by default, but if you would like to provide a custom path you can use the following configuration.
 
 ### Config
 Inside your `best.config.js` you need to have the following:
 ```
 {
     apiDatabase: {
-        adapter: 'sql/postgres',
-        path: 'postgresql://dbuser:secretpassword@database.server.com:3211/mydb
+        adapter: 'sql/sqlite',
+        uri: 'PATH_TO_SQLITE_DB'
     }
 }
 ```
 
-For Postgres, you need to provision and manage your own database.
-
-### Migrations
-In order to run the migrations required for the database you can run the following command:
-
-```
-yarn migrate:postgres up
-```
+You do not need to create your own sqlite file, the adapter will handle that for you.
 
-## SQLite
+## Postgres
 
 ### Config
 Inside your `best.config.js` you need to have the following:
 ```
 {
     apiDatabase: {
-        adapter: 'sql/sqlite',
-        path: 'PATH_TO_SQLITE_DB'
+        adapter: 'sql/postgres',
+        uri: 'postgresql://dbuser:secretpassword@database.server.com:3211/mydb
     }
 }
 ```
 
-You do not need to create your own sqlite file, the adapter will handle that for you.
+For Postgres, you need to provision and manage your own database.

packages/@best/api-db/package.json

@@ -2,15 +2,11 @@
     "name": "@best/api-db",
     "version": "4.0.0",
     "dependencies": {
-        "node-pg-migrate": "^3.21.1",
         "pg": "^7.11.0",
         "sqlite": "^3.0.3"
     },
     "devDependencies": {
         "@types/pg": "^7.4.14"
     },
-    "main": "build/index.js",
-    "scripts": {
-        "migrate:postgres": "node-pg-migrate -m src/sql/postgres/migrations"
-    }
+    "main": "build/index.js"
 }

packages/@best/api-db/src/sql/adapter.ts

@@ -23,42 +23,36 @@ export class SQLAdapter extends ApiDBAdapter {
     }
 
     async saveSnapshots(snapshots: TemporarySnapshot[], projectName: string): Promise<boolean> {
-        try {
-            let projectResult = await this.db.fetchProject(projectName)
+        let projectResult = await this.db.fetchProject(projectName)
 
-            if (projectResult.rows.length < 1) {
-                await this.db.createProject(projectName)
-                projectResult = await this.db.fetchProject(projectName)
-            }
+        if (projectResult.rows.length < 1) {
+            await this.db.createProject(projectName, true)
+            projectResult = await this.db.fetchProject(projectName)
+        }
 
-            const projectId = projectResult.rows[0].id
+        const projectId = projectResult.rows[0].id
 
-            await Promise.all(snapshots.map(async (snapshot) => {
-                return this.db.createOrUpdateSnapshot(snapshot, projectId)
-            }))
-        } catch (err) {
-            console.error('[API-DB] Could not save results into database.')
-            return false
-        }
+        await Promise.all(snapshots.map(async (snapshot) => {
+            return this.db.createOrUpdateSnapshot(snapshot, projectId)
+        }))
 
         return true
     }
 
     async updateLastRelease(projectName: string, release: string | Date): Promise<boolean> {
-        try {
-            const projectResult = await this.db.fetchProject(projectName)
+        const projectResult = await this.db.fetchProject(projectName)
 
-            if (projectResult.rows.length > 0) {
-                const projectId = projectResult.rows[0].id
-                await this.db.updateProjectLastRelease(projectId, release)
-            } else {
-                throw new Error(`Project with name: '${projectName}' does not exist.`)
-            }
-        } catch (err) {
-            console.log('[API-DB] Could not update latest result')
-            return false
+        if (projectResult.rows.length > 0) {
+            const projectId = projectResult.rows[0].id
+            await this.db.updateProjectLastRelease(projectId, release)
+        } else {
+            throw new Error(`Project with name: '${projectName}' does not exist.`)
         }
 
         return true
     }
+
+    migrate() {
+        return this.db.performMigrations()
+    }
 }

packages/@best/api-db/src/sql/db.ts

@@ -34,8 +34,16 @@ export abstract class SQLDatabase {
         return this.query('SELECT * FROM projects WHERE "name" = $1 LIMIT 1', [name])
     }
 
-    createProject(name: string): Promise<SQLQueryResult> {
-        return this.query('INSERT INTO projects("name") VALUES ($1)', [name])
+    async createProject(name: string, swallowNonUniqueErrors: boolean = false): Promise<SQLQueryResult> {
+        try {
+            return await this.query('INSERT INTO projects("name") VALUES ($1)', [name]);
+        } catch (err) {
+            if (swallowNonUniqueErrors && (err.constraint === 'projects_unique_name' || err.code === 'SQLITE_CONSTRAINT')) {
+                return this.fetchProject(name);
+            }
+
+            throw err;
+        }
     }
 
     updateProjectLastRelease(id: number, release: string | Date): Promise<SQLQueryResult> {
@@ -49,16 +57,19 @@ export abstract class SQLDatabase {
 
     async createOrUpdateSnapshot(snapshot: TemporarySnapshot, projectId: number): Promise<SQLQueryResult> {
         try {
-            return await this.createSnapshot(snapshot, projectId)
+            return await this.createSnapshot(snapshot, projectId);
         } catch (err) {
-            if (err.constraint === 'best_snapshot_unqiue_index') {
+            if (err.constraint === 'best_snapshot_unqiue_index' || err.code === 'SQLITE_CONSTRAINT') {
                 const updatedAt = new Date()
                 const values = [normalizeMetrics(snapshot.metrics), snapshot.environmentHash, snapshot.similarityHash, updatedAt, projectId, snapshot.commit, snapshot.name]
                 return this.query('UPDATE snapshots SET "metrics" = $1, "environment_hash" = $2, "similarity_hash" = $3, "updated_at" = $4 WHERE "project_id" = $5 AND "commit" = $6 AND "name" = $7', values)
-            } else {
-                console.log(err)
-                throw err;
             }
+
+            throw err;
         }
     }
+
+    async performMigrations() {
+        throw new Error('Migrations are not implemented.')
+    }
 }

packages/@best/api-db/src/sql/postgres/db.ts

@@ -1,17 +1,27 @@
 import { Pool } from 'pg'
 import { SQLDatabase, SQLQueryResult } from '../db'
 import { ApiDatabaseConfig } from '@best/types';
+import { migrate } from './migrate';
 
 export default class PostgresDatabase extends SQLDatabase {
     pool: Pool
+    migrated = false;
+
     constructor(config: ApiDatabaseConfig) {
         super()
         this.pool = new Pool({
-            connectionString: config.path
+            connectionString: config.uri
         })
     }
 
     query(text: string, params: any[]): Promise<SQLQueryResult> {
+        if (! this.migrated) { throw new Error('Database migrations have not been ensured.') }
+
         return this.pool.query(text, params)
     }
+
+    async performMigrations() {
+        await migrate(this.pool)
+        this.migrated = true;
+    }
 }

packages/@best/api-db/src/sql/postgres/migrate.ts

@@ -0,0 +1,93 @@
+import path from 'path'
+import fs from 'fs';
+import { promisify } from 'util';
+import { Pool } from 'pg';
+
+const asyncReadDir = promisify(fs.readdir);
+
+interface MigrationContent {
+    up: string;
+    down: string;
+}
+
+interface PartialMigration {
+    id: number;
+    name: string;
+    filename: string;
+}
+
+type Migration = PartialMigration & MigrationContent
+
+const buildMigrations = async (location: string): Promise<Migration[]> => {
+    const migrationsPath = path.resolve(__dirname, location);
+
+    const files = await asyncReadDir(migrationsPath);
+
+    // we look for .js files since these will be pre-compiled by js
+    const partialMigrations: PartialMigration[] = files.map(f => f.match(/^((\d+).(.*?))\.js$/)).reduce((migrations, match): PartialMigration[] => {
+        if (! match) {
+            return migrations;
+        }
+
+        const migration = { id: Number(match[2]), name: match[3], filename: match[1] }
+
+        return [...migrations, migration];
+    }, <PartialMigration[]>[]).sort((a, b) => Math.sign(a.id - b.id));
+
+    const migrations: Migration[] = await Promise.all(partialMigrations.map(async (partial): Promise<Migration> => {
+        const filename = path.resolve(migrationsPath, partial.filename);
+        const content: MigrationContent = await import(filename);
+
+        return {
+            ...partial,
+            ...content
+        }
+    }))
+
+    return migrations
+}
+
+export const migrate = async (db: Pool, redoLast: boolean = false, location: string = 'migrations/', table = 'migrations'): Promise<boolean> => {
+    const migrations = await buildMigrations(location);
+
+    const client = await db.connect();
+
+    await client.query(`CREATE TABLE IF NOT EXISTS "${table}" (id INTEGER PRIMARY KEY, name TEXT NOT NULL, up TEXT NOT NULL, down TEXT NOT NULL)`);
+
+    const previous = await client.query(`SELECT * FROM ${table} ORDER BY id ASC`);
+    let lastMigration = previous.rows[previous.rows.length - 1];
+
+    if (redoLast && previous.rows.length > 0) {
+        await client.query('BEGIN');
+        try {
+            await client.query(lastMigration.down);
+            await client.query(`DELETE FROM "${table}" WHERE id = ?`, lastMigration.id);
+            await client.query('COMMIT');
+            lastMigration = null;
+        } catch (err) {
+            await client.query('ROLLBACK');
+            client.release();
+            throw err;
+        }
+    }
+
+    const lastMigrationId = lastMigration ? lastMigration.id : 0;
+    await Promise.all(migrations.map(async migration => {
+        if (migration.id > lastMigrationId) {
+            await client.query('BEGIN');
+            try {
+                await client.query(migration.up);
+                await client.query(`INSERT INTO "${table}" (id, name, up, down) VALUES ($1, $2, $3, $4)`, [migration.id, migration.name, migration.up, migration.down]);
+                await client.query('COMMIT');
+            } catch (err) {
+                await client.query('ROLLBACK');
+                client.release();
+                throw err;
+            }
+        }
+    }))
+
+    client.release();
+
+    return true;
+}

packages/@best/api-db/src/sql/postgres/migrations/001-projects.ts

@@ -0,0 +1,12 @@
+export const up = `
+CREATE TABLE projects (
+    id SERIAL PRIMARY KEY,
+    name character varying(100) NOT NULL,
+    created_at timestamp without time zone NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    last_release_date timestamp without time zone
+);
+
+CREATE UNIQUE INDEX projects_unique_name ON projects(name text_ops);
+`
+
+export const down = `DROP TABLE projects;`

packages/@best/api-db/src/sql/postgres/migrations/002-snapshots.ts

@@ -0,0 +1,20 @@
+export const up = `
+CREATE TABLE snapshots (
+    id SERIAL PRIMARY KEY,
+    project_id integer NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
+    name character varying(200) NOT NULL,
+    metrics character varying(2000) NOT NULL,
+    environment_hash character varying(100) NOT NULL,
+    similarity_hash character varying(100) NOT NULL,
+    commit character varying(100) NOT NULL,
+    commit_date timestamp without time zone NOT NULL,
+    created_at timestamp without time zone NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    temporary boolean NOT NULL,
+    updated_at timestamp without time zone NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX snapshots_project_id_index ON snapshots(project_id int4_ops);
+CREATE UNIQUE INDEX best_snapshot_unqiue_index ON snapshots(project_id int4_ops,commit text_ops,name text_ops) WHERE temporary = false;
+`
+
+export const down = `DROP TABLE snapshots;`