Feature/add wait status check to pg upgrade (#1488)

* Clarify development entrypoint script * Check to ensure database is ready for promotion * Add --force flag for pg:promote * Only alert user to check on error * Update existing test to check for wait status * Make error message more severe * Test promotion when database is unavailable * Fix linting errors
heroku · May 4, 2020 · 16400a2 · 16400a2
1 parent fe7cc1d
commit 16400a2
Show file tree

Hide file tree

Showing 4 changed files with 207 additions and 55 deletions.
diff --git a/docs/pg.md b/docs/pg.md
@@ -3,54 +3,55 @@
 
 manage postgresql databases
 
-* [`heroku pg [DATABASE]`](#heroku-pg-database)
-* [`heroku pg:backups`](#heroku-pgbackups)
-* [`heroku pg:backups:cancel [BACKUP_ID]`](#heroku-pgbackupscancel-backup_id)
-* [`heroku pg:backups:capture [DATABASE]`](#heroku-pgbackupscapture-database)
-* [`heroku pg:backups:delete BACKUP_ID`](#heroku-pgbackupsdelete-backup_id)
-* [`heroku pg:backups:download [BACKUP_ID]`](#heroku-pgbackupsdownload-backup_id)
-* [`heroku pg:backups:info [BACKUP_ID]`](#heroku-pgbackupsinfo-backup_id)
-* [`heroku pg:backups:restore [BACKUP] [DATABASE]`](#heroku-pgbackupsrestore-backup-database)
-* [`heroku pg:backups:schedule [DATABASE]`](#heroku-pgbackupsschedule-database)
-* [`heroku pg:backups:schedules`](#heroku-pgbackupsschedules)
-* [`heroku pg:backups:unschedule [DATABASE]`](#heroku-pgbackupsunschedule-database)
-* [`heroku pg:backups:url [BACKUP_ID]`](#heroku-pgbackupsurl-backup_id)
-* [`heroku pg:bloat [DATABASE]`](#heroku-pgbloat-database)
-* [`heroku pg:blocking [DATABASE]`](#heroku-pgblocking-database)
-* [`heroku pg:connection-pooling:attach [DATABASE]`](#heroku-pgconnection-poolingattach-database)
-* [`heroku pg:copy SOURCE TARGET`](#heroku-pgcopy-source-target)
-* [`heroku pg:credentials [DATABASE]`](#heroku-pgcredentials-database)
-* [`heroku pg:credentials:create [DATABASE]`](#heroku-pgcredentialscreate-database)
-* [`heroku pg:credentials:destroy [DATABASE]`](#heroku-pgcredentialsdestroy-database)
-* [`heroku pg:credentials:repair-default [DATABASE]`](#heroku-pgcredentialsrepair-default-database)
-* [`heroku pg:credentials:rotate [DATABASE]`](#heroku-pgcredentialsrotate-database)
-* [`heroku pg:credentials:url [DATABASE]`](#heroku-pgcredentialsurl-database)
-* [`heroku pg:diagnose [DATABASE|REPORT_ID]`](#heroku-pgdiagnose-databasereport_id)
-* [`heroku pg:info [DATABASE]`](#heroku-pginfo-database)
-* [`heroku pg:kill PID [DATABASE]`](#heroku-pgkill-pid-database)
-* [`heroku pg:killall [DATABASE]`](#heroku-pgkillall-database)
-* [`heroku pg:links [DATABASE]`](#heroku-pglinks-database)
-* [`heroku pg:links:create REMOTE DATABASE`](#heroku-pglinkscreate-remote-database)
-* [`heroku pg:links:destroy DATABASE LINK`](#heroku-pglinksdestroy-database-link)
-* [`heroku pg:locks [DATABASE]`](#heroku-pglocks-database)
-* [`heroku pg:maintenance [DATABASE]`](#heroku-pgmaintenance-database)
-* [`heroku pg:maintenance:run [DATABASE]`](#heroku-pgmaintenancerun-database)
-* [`heroku pg:maintenance:window DATABASE WINDOW`](#heroku-pgmaintenancewindow-database-window)
-* [`heroku pg:outliers [DATABASE]`](#heroku-pgoutliers-database)
-* [`heroku pg:promote DATABASE`](#heroku-pgpromote-database)
-* [`heroku pg:ps [DATABASE]`](#heroku-pgps-database)
-* [`heroku pg:psql [DATABASE]`](#heroku-pgpsql-database)
-* [`heroku pg:pull SOURCE TARGET`](#heroku-pgpull-source-target)
-* [`heroku pg:push SOURCE TARGET`](#heroku-pgpush-source-target)
-* [`heroku pg:reset [DATABASE]`](#heroku-pgreset-database)
-* [`heroku pg:settings [DATABASE]`](#heroku-pgsettings-database)
-* [`heroku pg:settings:log-lock-waits [VALUE] [DATABASE]`](#heroku-pgsettingslog-lock-waits-value-database)
-* [`heroku pg:settings:log-min-duration-statement [VALUE] [DATABASE]`](#heroku-pgsettingslog-min-duration-statement-value-database)
-* [`heroku pg:settings:log-statement [VALUE] [DATABASE]`](#heroku-pgsettingslog-statement-value-database)
-* [`heroku pg:unfollow DATABASE`](#heroku-pgunfollow-database)
-* [`heroku pg:upgrade [DATABASE]`](#heroku-pgupgrade-database)
-* [`heroku pg:vacuum-stats [DATABASE]`](#heroku-pgvacuum-stats-database)
-* [`heroku pg:wait [DATABASE]`](#heroku-pgwait-database)
+- [`heroku pg`](#heroku-pg)
+  - [`heroku pg [DATABASE]`](#heroku-pg-database)
+  - [`heroku pg:backups`](#heroku-pgbackups)
+  - [`heroku pg:backups:cancel [BACKUP_ID]`](#heroku-pgbackupscancel-backupid)
+  - [`heroku pg:backups:capture [DATABASE]`](#heroku-pgbackupscapture-database)
+  - [`heroku pg:backups:delete BACKUP_ID`](#heroku-pgbackupsdelete-backupid)
+  - [`heroku pg:backups:download [BACKUP_ID]`](#heroku-pgbackupsdownload-backupid)
+  - [`heroku pg:backups:info [BACKUP_ID]`](#heroku-pgbackupsinfo-backupid)
+  - [`heroku pg:backups:restore [BACKUP] [DATABASE]`](#heroku-pgbackupsrestore-backup-database)
+  - [`heroku pg:backups:schedule [DATABASE]`](#heroku-pgbackupsschedule-database)
+  - [`heroku pg:backups:schedules`](#heroku-pgbackupsschedules)
+  - [`heroku pg:backups:unschedule [DATABASE]`](#heroku-pgbackupsunschedule-database)
+  - [`heroku pg:backups:url [BACKUP_ID]`](#heroku-pgbackupsurl-backupid)
+  - [`heroku pg:bloat [DATABASE]`](#heroku-pgbloat-database)
+  - [`heroku pg:blocking [DATABASE]`](#heroku-pgblocking-database)
+  - [`heroku pg:connection-pooling:attach [DATABASE]`](#heroku-pgconnection-poolingattach-database)
+  - [`heroku pg:copy SOURCE TARGET`](#heroku-pgcopy-source-target)
+  - [`heroku pg:credentials [DATABASE]`](#heroku-pgcredentials-database)
+  - [`heroku pg:credentials:create [DATABASE]`](#heroku-pgcredentialscreate-database)
+  - [`heroku pg:credentials:destroy [DATABASE]`](#heroku-pgcredentialsdestroy-database)
+  - [`heroku pg:credentials:repair-default [DATABASE]`](#heroku-pgcredentialsrepair-default-database)
+  - [`heroku pg:credentials:rotate [DATABASE]`](#heroku-pgcredentialsrotate-database)
+  - [`heroku pg:credentials:url [DATABASE]`](#heroku-pgcredentialsurl-database)
+  - [`heroku pg:diagnose [DATABASE|REPORT_ID]`](#heroku-pgdiagnose-databasereportid)
+  - [`heroku pg:info [DATABASE]`](#heroku-pginfo-database)
+  - [`heroku pg:kill PID [DATABASE]`](#heroku-pgkill-pid-database)
+  - [`heroku pg:killall [DATABASE]`](#heroku-pgkillall-database)
+  - [`heroku pg:links [DATABASE]`](#heroku-pglinks-database)
+  - [`heroku pg:links:create REMOTE DATABASE`](#heroku-pglinkscreate-remote-database)
+  - [`heroku pg:links:destroy DATABASE LINK`](#heroku-pglinksdestroy-database-link)
+  - [`heroku pg:locks [DATABASE]`](#heroku-pglocks-database)
+  - [`heroku pg:maintenance [DATABASE]`](#heroku-pgmaintenance-database)
+  - [`heroku pg:maintenance:run [DATABASE]`](#heroku-pgmaintenancerun-database)
+  - [`heroku pg:maintenance:window DATABASE WINDOW`](#heroku-pgmaintenancewindow-database-window)
+  - [`heroku pg:outliers [DATABASE]`](#heroku-pgoutliers-database)
+  - [`heroku pg:promote DATABASE`](#heroku-pgpromote-database)
+  - [`heroku pg:ps [DATABASE]`](#heroku-pgps-database)
+  - [`heroku pg:psql [DATABASE]`](#heroku-pgpsql-database)
+  - [`heroku pg:pull SOURCE TARGET`](#heroku-pgpull-source-target)
+  - [`heroku pg:push SOURCE TARGET`](#heroku-pgpush-source-target)
+  - [`heroku pg:reset [DATABASE]`](#heroku-pgreset-database)
+  - [`heroku pg:settings [DATABASE]`](#heroku-pgsettings-database)
+  - [`heroku pg:settings:log-lock-waits [VALUE] [DATABASE]`](#heroku-pgsettingslog-lock-waits-value-database)
+  - [`heroku pg:settings:log-min-duration-statement [VALUE] [DATABASE]`](#heroku-pgsettingslog-min-duration-statement-value-database)
+  - [`heroku pg:settings:log-statement [VALUE] [DATABASE]`](#heroku-pgsettingslog-statement-value-database)
+  - [`heroku pg:unfollow DATABASE`](#heroku-pgunfollow-database)
+  - [`heroku pg:upgrade [DATABASE]`](#heroku-pgupgrade-database)
+  - [`heroku pg:vacuum-stats [DATABASE]`](#heroku-pgvacuum-stats-database)
+  - [`heroku pg:wait [DATABASE]`](#heroku-pgwait-database)
 
 ## `heroku pg [DATABASE]`
 
@@ -585,6 +586,7 @@ USAGE
 OPTIONS
   -a, --app=app        (required) app to run command against
   -r, --remote=remote  git remote of app to use
+  -f, --force          skip database availability checks and force promotion
 ```
 
 ## `heroku pg:ps [DATABASE]`

diff --git a/packages/cli/README.md b/packages/cli/README.md
@@ -85,6 +85,8 @@ Developing
 
 This project is built with [lerna](https://lerna.js.org/). The core plugins are located in [./packages](./packages). Run `lerna bootstrap` after cloning the repository to set it up.
 
+The standard `oclif` `./bin/run` script serves as your entry point to the CLI in your local development environment.
+
 Releasing
 =========
 1. Checkout the master branch and double-check you're on latest commit that you would like to release from.

diff --git a/packages/pg-v5/commands/promote.js b/packages/pg-v5/commands/promote.js
@@ -2,11 +2,14 @@
 
 const cli = require('heroku-cli-util')
 const co = require('co')
+const host = require('../lib/host')
 
 function * run (context, heroku) {
   const fetcher = require('../lib/fetcher')(heroku)
-  const { app, args } = context
+  const { app, args, flags } = context
+  const { force } = flags
   const attachment = yield fetcher.attachment(app, args.database)
+
   let current
 
   yield cli.action(`Ensuring an alternate alias for existing ${cli.color.configVar('DATABASE_URL')}`, co(function * () {
@@ -45,6 +48,19 @@ function * run (context, heroku) {
     cli.action.done(cli.color.configVar(backup.name + '_URL'))
   }))
 
+  if (!force) {
+    let status = yield heroku.request({
+      host: host(attachment.addon),
+      path: `/client/v11/databases/${attachment.addon.id}/wait_status`
+    })
+
+    if (status['waiting?']) {
+      throw new Error(`Database cannot be promoted while in state: ${status['message']}
+\nPromoting this database can lead to application errors and outage. Please run pg:wait to wait for database to become available.
+\nTo ignore this error, you can pass the --force flag to promote the database and risk application issues.`)
+    }
+  }
+
   let promotionMessage
   if (attachment.namespace) {
     promotionMessage = `Promoting ${cli.color.attachment(attachment.name)} to ${cli.color.configVar('DATABASE_URL')} on ${cli.color.app(app)}`
@@ -120,6 +136,7 @@ module.exports = {
   description: 'sets DATABASE as your DATABASE_URL',
   needsApp: true,
   needsAuth: true,
+  flags: [{ name: 'force', char: 'f' }],
   args: [{ name: 'database' }],
   run: cli.command({ preauth: true }, co.wrap(run))
 }
diff --git a/packages/pg-v5/test/commands/promote.js b/packages/pg-v5/test/commands/promote.js
@@ -8,10 +8,12 @@ const proxyquire = require('proxyquire')
 
 describe('pg:promote when argument is database', () => {
   let api
+  let pg
 
   const attachment = {
     addon: {
-      name: 'postgres-1'
+      name: 'postgres-1',
+      id: 'c667bce0-3238-4202-8550-e1dc323a02a2'
     },
     namespace: null
   }
@@ -22,19 +24,27 @@ describe('pg:promote when argument is database', () => {
     }
   }
 
+  const host = () => {
+    return 'https://postgres-api.heroku.com'
+  }
+
   const cmd = proxyquire('../../commands/promote', {
-    '../lib/fetcher': fetcher
+    '../lib/fetcher': fetcher,
+    '../lib/host': host
   })
 
   beforeEach(() => {
     api = nock('https://api.heroku.com:443')
     api.get('/apps/myapp/formation').reply(200, [])
+    pg = nock('https://postgres-api.heroku.com')
+    pg.get(`/client/v11/databases/${attachment.addon.id}/wait_status`).reply(200, { message: 'available', 'waiting?': false })
     cli.mockConsole()
   })
 
   afterEach(() => {
     nock.cleanAll()
     api.done()
+    pg.done()
   })
 
   it('promotes the db and creates another attachment if current DATABASE does not have another', () => {
@@ -91,7 +101,10 @@ Promoting postgres-1 to DATABASE_URL on myapp... done
 describe('pg:promote when argument is a credential attachment', () => {
   const credentialAttachment = {
     name: 'PURPLE',
-    addon: { name: 'postgres-1' },
+    addon: {
+      name: 'postgres-1',
+      id: 'c667bce0-3238-4202-8550-e1dc323a02a2'
+    },
     namespace: 'credential:hello'
   }
 
@@ -101,21 +114,30 @@ describe('pg:promote when argument is a credential attachment', () => {
     }
   }
 
+  const host = () => {
+    return 'https://postgres-api.heroku.com'
+  }
+
   const cmd = proxyquire('../../commands/promote', {
-    '../lib/fetcher': fetcher
+    '../lib/fetcher': fetcher,
+    '../lib/host': host
   })
 
   let api
+  let pg
 
   beforeEach(() => {
     api = nock('https://api.heroku.com:443')
     api.get('/apps/myapp/formation').reply(200, [])
+    pg = nock('https://postgres-api.heroku.com')
+    pg.get(`/client/v11/databases/${credentialAttachment.addon.id}/wait_status`).reply(200, { message: 'available', 'waiting?': false })
     cli.mockConsole()
   })
 
   afterEach(() => {
     nock.cleanAll()
     api.done()
+    pg.done()
   })
 
   it('promotes the credential and creates another attachment if current DATABASE does not have another', () => {
@@ -254,8 +276,16 @@ Promoting PURPLE to DATABASE_URL on myapp... done
 
 describe('pg:promote when release phase is present', () => {
   let api
+  let pg
 
-  const cmd = proxyquire('../../commands/promote', {})
+  const addonID = 'c667bce0-3238-4202-8550-e1dc323a02a2'
+  const host = () => {
+    return 'https://postgres-api.heroku.com'
+  }
+
+  const cmd = proxyquire('../../commands/promote', {
+    '../lib/host': host
+  })
 
   beforeEach(() => {
     api = nock('https://api.heroku.com:443')
@@ -285,15 +315,19 @@ describe('pg:promote when release phase is present', () => {
       addon_service: 'heroku-postgresql'
     }).reply(201, [{
       name: 'PURPLE',
-      addon: { name: 'postgres-1' },
+      addon: { name: 'postgres-1', id: addonID },
       namespace: 'credential:hello'
     }])
 
+    pg = nock('https://postgres-api.heroku.com')
+    pg.get(`/client/v11/databases/${addonID}/wait_status`).reply(200, { message: 'available', 'waiting?': false })
+
     cli.mockConsole()
   })
 
   afterEach(() => {
     nock.cleanAll()
+    pg.done()
     api.done()
   })
 
@@ -346,3 +380,100 @@ Checking release phase... pg:promote failed because Attach DATABASE release was
     return expect(cmd.run({ app: 'myapp', args: {}, flags: {} }), 'to be rejected')
   })
 })
+
+describe('pg:promote when database is not available or force flag is present', () => {
+  let api
+  let pg
+
+  const attachment = {
+    addon: {
+      name: 'postgres-1',
+      id: 'c667bce0-3238-4202-8550-e1dc323a02a2'
+    },
+    namespace: null
+  }
+
+  const fetcher = () => {
+    return {
+      attachment: () => attachment
+    }
+  }
+
+  const host = () => {
+    return 'https://postgres-api.heroku.com'
+  }
+
+  const cmd = proxyquire('../../commands/promote', {
+    '../lib/fetcher': fetcher,
+    '../lib/host': host
+  })
+
+  beforeEach(() => {
+    api = nock('https://api.heroku.com:443')
+    api.get('/apps/myapp/formation').reply(200, [])
+    pg = nock('https://postgres-api.heroku.com')
+    cli.mockConsole()
+  })
+
+  afterEach(() => {
+    nock.cleanAll()
+    api.done()
+    pg.done()
+  })
+
+  it('warns user if database is unavailable', () => {
+    api.get('/apps/myapp/addon-attachments').reply(200, [
+      { name: 'DATABASE', addon: { name: 'postgres-2' }, namespace: null },
+      { name: 'RED', addon: { name: 'postgres-2' }, namespace: null }
+    ])
+
+    pg.get(`/client/v11/databases/${attachment.addon.id}/wait_status`).reply(200, { 'waiting?': true, message: 'pending' })
+
+    const err = new Error(`Database cannot be promoted while in state: pending
+\nPromoting this database can lead to application errors and outage. Please run pg:wait to wait for database to become available.
+\nTo ignore this error, you can pass the --force flag to promote the database and risk application issues.`)
+    return expect(cmd.run({ app: 'myapp', args: {}, flags: {} }), 'to be rejected with', err)
+  })
+
+  it('promotes database in unavailable state if --force flag is present', () => {
+    api.get('/apps/myapp/addon-attachments').reply(200, [
+      { name: 'DATABASE', addon: { name: 'postgres-2' }, namespace: null },
+      { name: 'RED', addon: { name: 'postgres-2' }, namespace: null }
+    ])
+
+    api.post('/addon-attachments', {
+      name: 'DATABASE',
+      app: { name: 'myapp' },
+      addon: { name: 'postgres-1' },
+      namespace: null,
+      confirm: 'myapp'
+    }).reply(201)
+
+    pg.get(`/client/v11/databases/${attachment.addon.id}/wait_status`).reply(200, { 'waiting?': true, message: 'pending' })
+
+    return cmd.run({ app: 'myapp', args: {}, flags: { force: true } })
+      .then(() => expect(cli.stderr, 'to equal', `Ensuring an alternate alias for existing DATABASE_URL... RED_URL
+Promoting postgres-1 to DATABASE_URL on myapp... done\n`))
+  })
+
+  it('promotes database in available state if --force flag is present', () => {
+    api.get('/apps/myapp/addon-attachments').reply(200, [
+      { name: 'DATABASE', addon: { name: 'postgres-2' }, namespace: null },
+      { name: 'RED', addon: { name: 'postgres-2' }, namespace: null }
+    ])
+
+    pg.get(`/client/v11/databases/${attachment.addon.id}/wait_status`).reply(200, { 'waiting?': false, message: 'available' })
+
+    api.post('/addon-attachments', {
+      name: 'DATABASE',
+      app: { name: 'myapp' },
+      addon: { name: 'postgres-1' },
+      namespace: null,
+      confirm: 'myapp'
+    }).reply(201)
+
+    return cmd.run({ app: 'myapp', args: {}, flags: { force: true } })
+      .then(() => expect(cli.stderr, 'to equal', `Ensuring an alternate alias for existing DATABASE_URL... RED_URL
+Promoting postgres-1 to DATABASE_URL on myapp... done\n`))
+  })
+})