feat: add healthchecks support

adonis-typings/database.ts

@@ -180,6 +180,15 @@ declare module '@ioc:Adonis/Lucid/Database' {
     port?: number,
   }
 
+  /**
+   * Shape of the report node for the database connection report
+   */
+  export type ReportNode = {
+    connection: string,
+    message: string,
+    error: any,
+  }
+
   /**
    * Shared config options for all clients
    */
@@ -188,6 +197,7 @@ declare module '@ioc:Adonis/Lucid/Database' {
     debug?: boolean,
     asyncStackTraces?: boolean,
     revision?: number,
+    healthCheck?: boolean,
     pool?: {
       afterCreate?: (conn: any, done: any) => void,
       min?: number,
@@ -476,6 +486,11 @@ declare module '@ioc:Adonis/Lucid/Database' {
      * re-add it using the `add` method
      */
     release (connectionName: string): Promise<void>
+
+    /**
+     * Returns the health check report for registered connections
+     */
+    report (): Promise<{ health: { healthy: boolean, message: string }, meta: ReportNode[] }>
   }
 
   /**
@@ -524,6 +539,11 @@ declare module '@ioc:Adonis/Lucid/Database' {
      * Disconnect knex
      */
     disconnect (): Promise<void>,
+
+    /**
+     * Returns the connection report
+     */
+    getReport (): Promise<ReportNode>
   }
 
   /**
@@ -611,6 +631,11 @@ declare module '@ioc:Adonis/Lucid/Database' {
      * Start a new transaction
      */
     transaction (): Promise<TransactionClientContract>
+
+    /**
+     * Returns the health check report for registered connections
+     */
+    report (): Promise<{ health: { healthy: boolean, message: string }, meta: ReportNode[] }>
   }
 
   const Database: DatabaseContract

providers/DatabaseProvider.ts

@@ -11,8 +11,8 @@ import { IocContract } from '@adonisjs/fold'
 
 import { Database } from '../src/Database'
 import { Adapter } from '../src/Orm/Adapter'
-import { column, computed } from '../src/Orm/Decorators'
 import { BaseModel } from '../src/Orm/BaseModel'
+import { column, computed } from '../src/Orm/Decorators'
 
 export default class DatabaseServiceProvider {
   constructor (protected $container: IocContract) {
@@ -43,4 +43,10 @@ export default class DatabaseServiceProvider {
       }
     })
   }
+
+  public boot () {
+    this.$container.with(['Adonis/Core/HealthCheck', 'Adonis/Lucid/Database'], (HealthCheck) => {
+      HealthCheck.addChecker('lucid', 'Adonis/Lucid/Database')
+    })
+  }
 }

src/Connection/Manager.ts

@@ -29,6 +29,10 @@ import { Connection } from './index'
 export class ConnectionManager extends EventEmitter implements ConnectionManagerContract {
   public connections: ConnectionManagerContract['connections'] = new Map()
 
+  /**
+   * Connections for which the config was patched. They must get removed
+   * overtime, unless application is behaving unstable.
+   */
   private _orphanConnections: Set<ConnectionContract> = new Set()
 
   constructor (private _logger: LoggerContract) {
@@ -240,4 +244,28 @@ export class ConnectionManager extends EventEmitter implements ConnectionManager
       this.connections.delete(connectionName)
     }
   }
+
+  /**
+   * Returns the report for all the connections marked for healthChecks.
+   */
+  public async report () {
+    const reports = await Promise.all(
+      Array.from(this.connections.keys())
+        .filter((one) => this.get(one)!.config.healthCheck)
+        .map((one) => {
+          this.connect(one)
+          return this.get(one)!.connection!.getReport()
+        }),
+      )
+
+    const healthy = !reports.find((report) => !!report.error)
+
+    return {
+      health: {
+        healthy,
+        message: healthy ? 'All connections are healthy' : 'One or more connections are not healthy',
+      },
+      meta: reports,
+    }
+  }
 }

src/Connection/index.ts

@@ -15,7 +15,7 @@ import { EventEmitter } from 'events'
 import { Exception } from '@poppinss/utils'
 import { patchKnex } from 'knex-dynamic-connection'
 import { LoggerContract } from '@ioc:Adonis/Core/Logger'
-import { ConnectionConfigContract, ConnectionContract } from '@ioc:Adonis/Lucid/Database'
+import { ConnectionConfigContract, ConnectionContract, ReportNode } from '@ioc:Adonis/Lucid/Database'
 
 /**
  * Connection class manages a given database connection. Internally it uses
@@ -243,6 +243,53 @@ export class Connection extends EventEmitter implements ConnectionContract {
     patchKnex(this.readClient, this._readConfigResolver.bind(this))
   }
 
+  /**
+   * Checks all the read hosts by running a query on them. Stops
+   * after first error.
+   */
+  private async _checkReadHosts () {
+    const configCopy = Object.assign({}, this.config)
+    let error: any = null
+
+    for (let _replica of this._readReplicas) {
+      configCopy.connection = this._readConfigResolver(this.config)
+      this._logger.trace({ connection: this.name }, 'spawing health check read connection')
+      const client = knex(configCopy)
+
+      try {
+        await client.raw('SELECT 1 + 1 AS result')
+      } catch (err) {
+        error = err
+      }
+
+      /**
+       * Cleanup client connection
+       */
+      await client.destroy()
+      this._logger.trace({ connection: this.name }, 'destroying health check read connection')
+
+      /**
+       * Return early when there is an error
+       */
+      if (error) {
+        break
+      }
+    }
+
+    return error
+  }
+
+  /**
+   * Checks for the write host
+   */
+  private async _checkWriteHost () {
+    try {
+      await this.client!.raw('SELECT 1 + 1 AS result')
+    } catch (error) {
+      return error
+    }
+  }
+
   /**
    * Returns the pool instance for the given connection
    */
@@ -314,4 +361,24 @@ export class Connection extends EventEmitter implements ConnectionContract {
       }
     }
   }
+
+  /**
+   * Returns the healthcheck report for the connection
+   */
+  public async getReport (): Promise<ReportNode> {
+    const error = await this._checkWriteHost()
+    let readError: Error | undefined
+
+    if (!error && this.hasReadWriteReplicas) {
+      readError = await this._checkReadHosts()
+    }
+
+    return {
+      connection: this.name,
+      message: readError
+        ? 'Unable to reach one of the read hosts'
+        : (error ? 'Unable to reach the database server' : 'Connection is healthy'),
+      error: error || readError || null,
+    }
+  }
 }

src/Database/index.ts

@@ -156,4 +156,11 @@ export class Database implements DatabaseContract {
   public raw (sql: string, bindings?: any, options?: DatabaseClientOptions) {
     return this.connection(this.primaryConnectionName, options).raw(sql, bindings)
   }
+
+  /**
+   * Invokes `manager.report`
+   */
+  public report () {
+    return this.manager.report()
+  }
 }

test/connection/connection-manager.spec.ts

@@ -190,4 +190,33 @@ test.group('ConnectionManager', (group) => {
     manager.patch('primary', getConfig())
     manager.connect('primary')
   })
+
+  test('get health check report for connections that has enabled health checks', async (assert) => {
+    const manager = new ConnectionManager(getLogger())
+    manager.add('primary', Object.assign({}, getConfig(), { healthCheck: true }))
+    manager.add('secondary', Object.assign({}, getConfig(), { healthCheck: true }))
+    manager.add('secondary-copy', Object.assign({}, getConfig(), { healthCheck: false }))
+
+    const report = await manager.report()
+    assert.equal(report.health.healthy, true)
+    assert.equal(report.health.message, 'All connections are healthy')
+    assert.lengthOf(report.meta, 2)
+    assert.deepEqual(report.meta.map(({ connection }) => connection), ['primary', 'secondary'])
+  })
+
+  test('get health check report when one of the connection is unhealthy', async (assert) => {
+    const manager = new ConnectionManager(getLogger())
+    manager.add('primary', Object.assign({}, getConfig(), { healthCheck: true }))
+    manager.add('secondary', Object.assign({}, getConfig(), {
+      healthCheck: true,
+      connection: { host: 'bad-host' },
+    }))
+    manager.add('secondary-copy', Object.assign({}, getConfig(), { healthCheck: false }))
+
+    const report = await manager.report()
+    assert.equal(report.health.healthy, false)
+    assert.equal(report.health.message, 'One or more connections are not healthy')
+    assert.lengthOf(report.meta, 2)
+    assert.deepEqual(report.meta.map(({ connection }) => connection), ['primary', 'secondary'])
+  }).timeout(0)
 })

test/connection/connection.spec.ts

@@ -142,21 +142,79 @@ test.group('Connection | setup', (group) => {
     const fn = () => connection.connect()
     assert.throw(fn, /knex: Required configuration option/)
   })
+
+  if (process.env.DB === 'mysql') {
+    test.group('Connection | setup mysql', () => {
+      test('pass user config to mysql driver', async (assert) => {
+        const config = getConfig() as MysqlConfigContract
+        config.connection!.charset = 'utf-8'
+        config.connection!.typeCast = false
+
+        const connection = new Connection('primary', config, getLogger())
+        connection.connect()
+
+        assert.equal(connection.client!['_context'].client.constructor.name, 'Client_MySQL')
+        assert.equal(connection.client!['_context'].client.config.connection.charset, 'utf-8')
+        assert.equal(connection.client!['_context'].client.config.connection.typeCast, false)
+      })
+    })
+  }
 })
 
-if (process.env.DB === 'mysql') {
-  test.group('Connection | setup mysql', () => {
-    test('pass user config to mysql driver', async (assert) => {
-      const config = getConfig() as MysqlConfigContract
-      config.connection!.charset = 'utf-8'
-      config.connection!.typeCast = false
+test.group('Health Checks', (group) => {
+  group.before(async () => {
+    await setup()
+  })
 
-      const connection = new Connection('primary', config, getLogger())
-      connection.connect()
+  group.after(async () => {
+    await cleanup()
+  })
 
-      assert.equal(connection.client!['_context'].client.constructor.name, 'Client_MySQL')
-      assert.equal(connection.client!['_context'].client.config.connection.charset, 'utf-8')
-      assert.equal(connection.client!['_context'].client.config.connection.typeCast, false)
+  test('get healthcheck report for healthy connection', async (assert) => {
+    const connection = new Connection('primary', getConfig(), getLogger())
+    connection.connect()
+
+    const report = await connection.getReport()
+    assert.deepEqual(report, {
+      connection: 'primary',
+      message: 'Connection is healthy',
+      error: null,
     })
   })
-}
+
+  if (process.env.DB !== 'sqlite') {
+    test('get healthcheck report for un-healthy connection', async (assert) => {
+      const connection = new Connection('primary', Object.assign({}, getConfig(), {
+        connection: {
+          host: 'bad-host',
+        },
+      }), getLogger())
+      connection.connect()
+
+      const report = await connection.getReport()
+      assert.equal(report.message, 'Unable to reach the database server')
+      assert.equal(report.error!.errno, 'ENOTFOUND')
+    }).timeout(0)
+
+    test('get healthcheck report for un-healthy read host', async (assert) => {
+      const connection = new Connection('primary', Object.assign({}, getConfig(), {
+        replicas: {
+          write: {
+            connection: getConfig().connection,
+          },
+          read: {
+            connection: [
+              getConfig().connection,
+              Object.assign({}, getConfig().connection, { host: 'bad-host' }),
+            ],
+          },
+        },
+      }), getLogger())
+      connection.connect()
+
+      const report = await connection.getReport()
+      assert.equal(report.message, 'Unable to reach one of the read hosts')
+      assert.equal(report.error!.errno, 'ENOTFOUND')
+    }).timeout(0)
+  }
+})