Expand typeorm usage

Successfully wrapped underlying typeorm connection to allow raw sql queries in the same node-mysql style. Migrations also now work. Automatic migration generation seems a little buggy (it always tries to recreate indexes). Not the end of the world-- most migrations inevitably need to be written manually anyway.

ormconfig.js

@@ -0,0 +1,16 @@
+const {DB_HOST, DB_NAME, DB_USER, DB_PASS, DB_PORT} = require('./dist/src/settings')
+
+module.exports = {
+    "type": "mysql",
+    "host": DB_HOST,
+    "port": DB_PORT,
+    "username": DB_USER,
+    "password": DB_PASS,
+    "database": DB_NAME,
+    "entities": ["dist/src/model/**/*.js"],
+    "migrations": ["dist/src/migration/**/*.js"],
+    "cli": {
+        "entitiesDir": "src/model",
+        "migrationsDir": "src/migration"
+    }
+ }

src/ChartBaker.tsx

@@ -33,7 +33,6 @@ export class ChartBaker {
         this.props = props
         this.baseDir = path.join(this.props.repoDir, BUILD_GRAPHER_PATH)
         fs.mkdirpSync(this.baseDir)
-        db.connect()
     }
 
     async bakeAssets() {

src/admin/api.ts

@@ -17,7 +17,6 @@ import {Request, Response, CurrentUser} from './authentication'
 import {getVariableData} from '../model/Variable'
 import { ChartConfigProps } from '../../js/charts/ChartConfig'
 import CountryNameFormat, { CountryDefByKey } from '../../js/standardizer/CountryNameFormat'
-import { uniq } from '../../js/charts/Util';
 
 // Little wrapper to automatically send returned objects as JSON, makes
 // the API code a bit cleaner
@@ -359,7 +358,6 @@ api.delete('/users/:userId', async (req: Request, res: Response) => {
     }
 
     await db.transaction(async t => {
-        await t.execute(`DELETE FROM user_invitations WHERE user_id=?`, req.params.userId)
         await t.execute(`DELETE FROM users WHERE id=?`, req.params.userId)
     })
 
@@ -393,7 +391,6 @@ api.post('/users/invite', async (req: Request, res: Response) => {
         invite.validTill = new Date(Date.now() + 7 * 24 * 60 * 60 * 1000)
         invite.created_at = new Date()
         invite.updated_at = new Date()
-        invite.user_id = res.locals.user.id
         await repo.save(invite)
 
         const inviteLink = absoluteUrl(`/admin/register?code=${invite.code}`)
@@ -899,7 +896,7 @@ api.post('/importDataset', async (req: Request, res: Response) => {
         }
 
         // Insert any new entities into the db
-        const entitiesUniq = uniq(entities)
+        const entitiesUniq = _.uniq(entities)
         const importEntityRows = entitiesUniq.map(e => [e, false, now, now, ""])
         await t.execute(`INSERT IGNORE entities (name, validated, created_at, updated_at, displayName) VALUES ?`, [importEntityRows])
 

src/bakeCharts.ts

@@ -1,4 +1,5 @@
 import {ChartBaker} from './ChartBaker'
+import * as db from './db'
 import * as parseArgs from 'minimist'
 import * as os from 'os'
 import * as path from 'path'
@@ -10,6 +11,7 @@ async function main(email: string, name: string, message: string) {
     })
 
     try {
+        await db.connect()
         await baker.bakeAll()
         await baker.deploy(message || "Automated update", email, name)
     } catch (err) {

src/db.ts

@@ -1,23 +1,15 @@
 import * as mysql from 'mysql'
-
+import * as typeorm from 'typeorm'
 import {DB_HOST, DB_NAME, DB_USER, DB_PASS, DB_PORT} from './settings'
-import {Connection, createConnection} from "typeorm"
 
 import {Chart} from './model/Chart'
 import User from './model/User'
 import UserInvitation from './model/UserInvitation'
 
-let pool: mysql.Pool
-let connection: Connection
+let connection: typeorm.Connection
 
 export async function connect() {
-    pool = mysql.createPool({
-        host: DB_HOST,
-        user: DB_USER,
-        database: DB_NAME
-    })
-
-    connection = await createConnection({
+    connection = await typeorm.createConnection({
         type: "mysql",
         host: DB_HOST,
         port: DB_PORT,
@@ -28,86 +20,42 @@ export async function connect() {
     })
 }
 
-export function getConnection(): Promise<mysql.PoolConnection> {
-    return new Promise((resolve, reject) => {
-        pool.getConnection((poolerr, conn) => {
-            if (poolerr) {
-                reject(poolerr)
-            } else {
-                resolve(conn)
-            }
-        })
-    })
-}
-
 class TransactionContext {
-    conn: mysql.PoolConnection
-    constructor(conn: mysql.PoolConnection) {
-        this.conn = conn
+    manager: typeorm.EntityManager
+    constructor(manager: typeorm.EntityManager) {
+        this.manager = manager
     }
 
     execute(queryStr: string, params?: any[]): Promise<any> {
-        return new Promise((resolve, reject) => {
-            this.conn.query(queryStr, params, (err, rows) => {
-                if (err) return reject(err)
-                resolve(rows)
-            })
-        })
+        return this.manager.query(params ? mysql.format(queryStr, params) : queryStr)
     }
 
     query(queryStr: string, params?: any[]): Promise<any> {
-        return new Promise((resolve, reject) => {
-            this.conn.query(queryStr, params, (err, rows) => {
-                if (err) return reject(err)
-                resolve(rows)
-            })
-        })
+        return this.manager.query(params ? mysql.format(queryStr, params) : queryStr)
     }
 }
 
 export async function transaction<T>(callback: (t: TransactionContext) => Promise<T>): Promise<T> {
-    const conn = await getConnection()
-    const t = new TransactionContext(conn)
-
-    try {
-        await t.execute("START TRANSACTION")
-        const result = await callback(t)
-        await t.execute("COMMIT")
-        return result
-    } catch (err) {
-        await t.execute("ROLLBACK")
-        throw err
-    } finally {
-        t.conn.release()
-    }
+    return typeorm.getConnection().transaction(async manager => {
+        const t = new TransactionContext(manager)
+        await callback(t)
+    })
 }
 
-export function query(queryStr: string, params?: any[]): Promise<any> {
-    return new Promise((resolve, reject) => {
-        pool.query(queryStr, params, (err, rows) => {
-            if (err) return reject(err)
-            resolve(rows)
-        })
-    })
+export async function query(queryStr: string, params?: any[]): Promise<any> {
+    return typeorm.getConnection().query(params ? mysql.format(queryStr, params) : queryStr)
 }
 
 // For operations that modify data (TODO: handling to check query isn't used for this)
-export function execute(queryStr: string, params?: any[]): Promise<any> {
-    return new Promise((resolve, reject) => {
-        pool.query(queryStr, params, (err, rows) => {
-            if (err) return reject(err)
-            resolve(rows)
-        })
-    })
+export async function execute(queryStr: string, params?: any[]): Promise<any> {
+    return typeorm.getConnection().query(params ? mysql.format(queryStr, params) : queryStr)
 }
 
 export async function get(queryStr: string, params?: any[]): Promise<any> {
     return (await query(queryStr, params))[0]
 }
 
 export async function end() {
-    if (pool)
-        pool.end()
     if (connection)
         await connection.close()
 }

src/migration/1534413949262-Typeorm.ts

@@ -0,0 +1,33 @@
+import {MigrationInterface, QueryRunner} from "typeorm";
+
+export class Typeorm1534413949262 implements MigrationInterface {
+
+    public async up(queryRunner: QueryRunner): Promise<any> {
+        await queryRunner.query("ALTER TABLE `user_invitations` DROP FOREIGN KEY `user_invitations_user_id_29cac16b_fk_users_id`");
+        await queryRunner.query("ALTER TABLE `charts` DROP FOREIGN KEY `charts_last_edited_by_791cce39_fk_users_name`");
+        await queryRunner.query("ALTER TABLE `charts` DROP FOREIGN KEY `charts_published_by_e3f4abdf_fk_users_name`");
+        await queryRunner.query("ALTER TABLE `user_invitations` DROP COLUMN `user_id`");
+        await queryRunner.query("ALTER TABLE `users` DROP COLUMN `last_login`");
+        await queryRunner.query("ALTER TABLE `user_invitations` DROP COLUMN `status`");
+        await queryRunner.query("UPDATE users SET full_name='' WHERE full_name IS NULL");
+        await queryRunner.query("ALTER TABLE `users` CHANGE `full_name` `full_name` varchar(255) NOT NULL DEFAULT ''");
+        await queryRunner.query("ALTER TABLE `users` CHANGE `is_active` `is_active` tinyint NOT NULL DEFAULT 1");
+        await queryRunner.query("ALTER TABLE `users` CHANGE `is_superuser` `is_superuser` tinyint NOT NULL DEFAULT 0");
+        await queryRunner.query("ALTER TABLE `users` CHANGE `created_at` `created_at` datetime NOT NULL");
+        await queryRunner.query("ALTER TABLE `users` CHANGE `updated_at` `updated_at` datetime NOT NULL");
+        await queryRunner.query("ALTER TABLE `user_invitations` CHANGE `valid_till` `valid_till` datetime NOT NULL");
+        await queryRunner.query("ALTER TABLE `user_invitations` CHANGE `created_at` `created_at` datetime NOT NULL");
+        await queryRunner.query("ALTER TABLE `user_invitations` CHANGE `updated_at` `updated_at` datetime NOT NULL");
+        await queryRunner.query("ALTER TABLE `charts` CHANGE `last_edited_at` `last_edited_at` datetime NOT NULL");
+        await queryRunner.query("ALTER TABLE `charts` CHANGE `published_at` `published_at` datetime NULL");
+        await queryRunner.query("ALTER TABLE `charts` CHANGE `created_at` `created_at` datetime NOT NULL");
+        await queryRunner.query("ALTER TABLE `charts` CHANGE `updated_at` `updated_at` datetime NOT NULL");
+        await queryRunner.query("ALTER TABLE `charts` CHANGE `starred` `starred` tinyint NOT NULL");
+        await queryRunner.query("ALTER TABLE `charts` ADD CONSTRAINT `FK_ebe44242cb70398fcb5af3c9316` FOREIGN KEY (`last_edited_by`) REFERENCES `users`(`name`)");
+        await queryRunner.query("ALTER TABLE `charts` ADD CONSTRAINT `FK_b3879b5deca71fae207d0365257` FOREIGN KEY (`published_by`) REFERENCES `users`(`name`)");
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<any> {
+        throw new Error("Unsupported")
+    }
+}

src/model/Chart.ts

@@ -1,19 +1,27 @@
 import * as _ from 'lodash'
+import {Entity, PrimaryGeneratedColumn, Column, BaseEntity, ManyToOne, JoinColumn} from "typeorm"
 
 import * as db from '../db'
 import ChartConfig, { ChartConfigProps } from '../../js/charts/ChartConfig'
 import {getVariableData} from './Variable'
-
-
-import {Entity, PrimaryGeneratedColumn, Column, BaseEntity} from "typeorm"
+import User from './User'
 
 @Entity("charts")
 export class Chart extends BaseEntity {
-    @PrimaryGeneratedColumn()
-    id!: number
+    @PrimaryGeneratedColumn() id!: number
+    @Column({ type: 'json' }) config: any
+    @Column({ name: 'last_edited_at' }) lastEditedAt!: Date
+    @Column({ name: 'last_edited_by', nullable: true }) lastEditedByUserId!: string
+    @Column({ name: 'published_at', nullable: true }) publishedAt!: Date
+    @Column({ name: 'published_by', nullable: true }) publishedByUserId!: string
+    @Column({ name: 'created_at' }) createdAt!: Date
+    @Column({ name: 'updated_at' }) updatedAt!: Date
+    @Column() starred!: boolean
 
-    @Column({ type: 'json' })
-    config: any
+    @ManyToOne(type => User, user => user.lastEditedCharts) @JoinColumn({ name: 'last_edited_by', referencedColumnName: 'name' })
+    lastEditedByUser!: User
+    @ManyToOne(type => User, user => user.publishedCharts) @JoinColumn({ name: 'published_by', referencedColumnName: 'name' })
+    publishedByUser!: User
 }
 
 // TODO integrate this old logic with typeorm

src/model/User.ts

@@ -1,18 +1,25 @@
-import {Entity, PrimaryGeneratedColumn, Column, BaseEntity} from "typeorm"
+import {Entity, PrimaryGeneratedColumn, Column, BaseEntity, OneToMany} from "typeorm"
+import { Chart } from './Chart'
 const hashers = require('node-django-hashers')
 
 @Entity("users")
 export default class User extends BaseEntity {
     @PrimaryGeneratedColumn() id!: number
-    @Column() name!: string
-    @Column() email!: string
-    @Column({ name: 'password' }) cryptedPassword!: string
-    @Column({ name: 'full_name' }) fullName!: string
-    @Column({ name: 'is_active' }) isActive: boolean = true
-    @Column({ name: 'is_superuser' }) isSuperuser: boolean = false
+    @Column({ unique: true }) name!: string
+    @Column({ unique: true }) email!: string
+    @Column({ name: 'password', length: 128 }) cryptedPassword!: string
+    @Column({ name: 'full_name', default: "" }) fullName!: string
+    @Column({ name: 'is_active', default: true }) isActive!: boolean
+    @Column({ name: 'is_superuser', default: false }) isSuperuser!: boolean
     @Column() created_at!: Date
     @Column() updated_at!: Date
 
+    @OneToMany(type => Chart, chart => chart.lastEditedByUser)
+    lastEditedCharts!: Chart[]
+
+    @OneToMany(type => Chart, chart => chart.publishedByUser)
+    publishedCharts!: Chart[]
+
     async setPassword(password: string) {
         const h = new hashers.BCryptPasswordHasher()
         this.cryptedPassword = await h.encode(password)

src/model/UserInvitation.ts

@@ -15,8 +15,4 @@ export default class UserInvitation extends BaseEntity {
     // Hack - some weirdness going on with default values and typeorm
     @Column() created_at!: Date
     @Column() updated_at!: Date
-
-    // Deprecated
-    @Column() status: string = "pending"
-    @Column() user_id!: number
 }