feat(v2): update knex-migrator execution for Ghost 2.0 (#765)

refs #759

- We have moved the execution of knex-migrator into Ghost 2.0.0
- This commit will ensure we skip the db migration when you:
  - migrate from ^1 to ^2
  - you update from ^2 to ^2
  - when you install ^2
- Added net socket server for Ghost 2.0 (alternative to simple port polling)
- way better error handling between Ghost and the CLI
- Ghost 2.0 executes knex-migrator
  - it will turn maintenance on if migrations need to be executed
  - the handling of receiving success or failure state requires a better communication between the CLI and Ghost, because the blog stays in maintenance mode and runs the migrations in background
- Ghost will tell the CLI when it's ready by using an extension: write a socket url into the config and send the success/failure state
- this is much better than using the http socket to communicate, because
  - A) port polling connects to the http port, it's impossible to send simple messages over this transport layer
  - B) the code is much simpler, CLI opens a socket port and Ghost pushes a notification if the notification is available
  - C) we receive any error from Ghost - even if the http server wasn't started yet
- we don't communicate with Ghost, Ghost communicates with the CLI
- port polling for v1 blogs is untouched, still works as expected
- coverage has decreased a very little 0,2% - will try to add more tests when we merge the 1.9 branch into master

extensions/systemd/systemd.js

@@ -21,8 +21,27 @@ class SystemdProcessManager extends ProcessManager {
         this._precheck();
         const {logSuggestion} = this;
 
-        return this.ui.sudo(`systemctl start ${this.systemdName}`)
-            .then(() => this.ensureStarted({logSuggestion}))
+        const portfinder = require('portfinder');
+        const socketAddress = {
+            port: null,
+            host: 'localhost'
+        };
+
+        return portfinder.getPortPromise()
+            .then((port) => {
+                socketAddress.port = port;
+                this.instance.config.set('bootstrap-socket', socketAddress);
+                return this.instance.config.save();
+            })
+            .then(() => {
+                return this.ui.sudo(`systemctl start ${this.systemdName}`)
+            })
+            .then(() => {
+                return this.ensureStarted({
+                    logSuggestion,
+                    socketAddress
+                });
+            })
             .catch((error) => {
                 if (error instanceof CliError) {
                     throw error;
@@ -43,8 +62,27 @@ class SystemdProcessManager extends ProcessManager {
         this._precheck();
         const {logSuggestion} = this;
 
-        return this.ui.sudo(`systemctl restart ${this.systemdName}`)
-            .then(() => this.ensureStarted({logSuggestion}))
+        const portfinder = require('portfinder');
+        const socketAddress = {
+            port: null,
+            host: 'localhost'
+        };
+
+        return portfinder.getPortPromise()
+            .then((port) => {
+                socketAddress.port = port;
+                this.instance.config.set('bootstrap-socket', socketAddress);
+                return this.instance.config.save();
+            })
+            .then(() => {
+                return this.ui.sudo(`systemctl restart ${this.systemdName}`)
+            })
+            .then(() => {
+                return this.ensureStarted({
+                    logSuggestion,
+                    socketAddress
+                });
+            })
             .catch((error) => {
                 if (error instanceof CliError) {
                     throw error;

extensions/systemd/test/systemd-spec.js

@@ -5,6 +5,7 @@ const proxyquire = require('proxyquire').noCallThru();
 
 const modulePath = '../systemd';
 const errors = require('../../../lib/errors');
+const configStub = require('../../../test/utils/config-stub');
 const Systemd = require(modulePath);
 
 const instance = {
@@ -30,6 +31,7 @@ describe('Unit: Systemd > Process Manager', function () {
         let ext, ui;
 
         beforeEach(function () {
+            instance.config = configStub();
             ui = {sudo: sinon.stub().resolves()},
             ext = new Systemd(ui, null, instance);
             ext.ensureStarted = sinon.stub().resolves();

lib/commands/setup.js

@@ -41,6 +41,7 @@ class SetupCommand extends Command {
         const os = require('os');
         const url  = require('url');
         const path = require('path');
+        const semver = require('semver');
 
         const linux = require('../tasks/linux');
         const migrate = require('../tasks/migrate');
@@ -171,10 +172,20 @@ class SetupCommand extends Command {
             }));
 
             if (argv.migrate !== false) {
+                const instance = this.system.getInstance();
+
                 // Tack on db migration task to the end
                 tasks.push({
                     title: 'Running database migrations',
-                    task: migrate
+                    task: migrate,
+                    // CASE: We are about to install Ghost 2.0. We moved the execution of knex-migrator into Ghost.
+                    enabled: () => {
+                        if (semver.satisfies(instance.cliConfig.get('active-version'), '^2.0.0')) {
+                            return false;
+                        }
+
+                        return true;
+                    }
                 });
             }
 

lib/commands/update.js

@@ -73,7 +73,17 @@ class UpdateCommand extends Command {
             }, {
                 title: 'Running database migrations',
                 skip: (ctx) => ctx.rollback,
-                task: migrate
+                task: migrate,
+                // CASE: We have moved the execution of knex-migrator into Ghost 2.0.0.
+                //       If you are already on ^2 or you update from ^1 to ^2, then skip the task.
+                enabled: () => {
+                    if (semver.satisfies(instance.cliConfig.get('active-version'), '^2.0.0') ||
+                        semver.satisfies(context.version, '^2.0.0')) {
+                        return false;
+                    }
+
+                    return true;
+                }
             }, {
                 title: 'Restarting Ghost',
                 skip: () => !argv.restart,

lib/process-manager.js

@@ -70,11 +70,13 @@ class ProcessManager {
      */
     ensureStarted(options) {
         const portPolling = require('./utils/port-polling');
+        const semver = require('semver');
 
         options = Object.assign({
             stopOnError: true,
             port: this.instance.config.get('server.port'),
-            host: this.instance.config.get('server.host', 'localhost')
+            host: this.instance.config.get('server.host', 'localhost'),
+            useNetServer: semver.satisfies(this.instance.cliConfig.get('active-version'), '^2.0.0')
         }, options || {});
 
         return portPolling(options).catch((err) => {

lib/utils/port-polling.js

@@ -1,24 +1,79 @@
 'use strict';
 
-const net = require('net');
 const errors = require('../errors');
 
-module.exports = function portPolling(options) {
-    options = Object.assign({
-        timeoutInMS: 2000,
-        maxTries: 20,
-        delayOnConnectInMS: 3 * 2000,
-        logSuggestion: 'ghost log',
-        socketTimeoutInMS: 1000 * 60
-    }, options || {});
+/**
+ * @TODO: in theory it could happen that other clients connect, but tbh even with the port polling it was possible: you
+ *        could just start a server on the Ghost port
+ */
+const useNetServer = (options)=> {
+    return new Promise((resolve, reject)=> {
+        const net = require('net');
+        let waitTimeout = null;
+        let ghostSocket = null;
+
+        const server = net.createServer((socket)=> {
+            ghostSocket = socket;
+
+            socket.on('data', (data) => {
+                let message;
+
+                try {
+                    message = JSON.parse(data);
+                } catch (err) {
+                    message = {started: false, error: err};
+                }
+
+                if (waitTimeout) {
+                    clearTimeout(waitTimeout);
+                }
+
+                socket.destroy();
+                ghostSocket = null;
+
+                server.close(() => {
+                    if (message.started) {
+                        resolve();
+                    } else {
+                        reject(new errors.GhostError({
+                            message: message.error.message,
+                            help: message.error.help,
+                            suggestion: options.logSuggestion
+                        }));
+                    }
+                });
+            });
+        });
+
+        waitTimeout = setTimeout(() => {
+            if (ghostSocket) {
+                ghostSocket.destroy();
+            }
+
+            ghostSocket = null;
+
+            server.close(() => {
+                reject(new errors.GhostError({
+                    message: 'Could not communicate with Ghost',
+                    suggestion: options.logSuggestion
+                }));
+            });
+        }, options.netServerTimeoutInMS);
+
+        server.listen({host: options.socketAddress.host, port: options.socketAddress.port});
+    });
+};
+
+const usePortPolling = (options)=> {
+    const net = require('net');
 
     if (!options.port) {
         return Promise.reject(new errors.CliError({
             message: 'Port is required.'
         }));
     }
 
-    const connectToGhostSocket = (() => {
+    const connectToGhostSocket = () => {
         return new Promise((resolve, reject) => {
             // if host is specified and is *not* 0.0.0.0 (listen on all ips), use the custom host
             const host = options.host && options.host !== '0.0.0.0' ? options.host : 'localhost';
@@ -35,7 +90,7 @@ module.exports = function portPolling(options) {
                 ghostSocket.destroy();
 
                 // force retry
-                const err = new Error();
+                const err = new Error('Socket timed out.');
                 err.retry = true;
                 reject(err);
             }));
@@ -73,7 +128,7 @@ module.exports = function portPolling(options) {
                 reject(err);
             }));
         });
-    });
+    };
 
     const startPolling = (() => {
         return new Promise((resolve, reject) => {
@@ -87,10 +142,14 @@ module.exports = function portPolling(options) {
                     .catch((err) => {
                         if (err.retry && tries < options.maxTries) {
                             tries = tries + 1;
-                            setTimeout(retry, options.timeoutInMS);
+                            setTimeout(retry, options.retryTimeoutInMS);
                             return;
                         }
 
+                        if (err instanceof errors.CliError) {
+                            return reject(err);
+                        }
+
                         reject(new errors.GhostError({
                             message: 'Ghost did not start.',
                             suggestion: options.logSuggestion,
@@ -103,3 +162,25 @@ module.exports = function portPolling(options) {
 
     return startPolling();
 };
+
+module.exports = function portPolling(options) {
+    options = Object.assign({
+        retryTimeoutInMS: 2000,
+        maxTries: 20,
+        delayOnConnectInMS: 3 * 2000,
+        logSuggestion: 'ghost log',
+        socketTimeoutInMS: 1000 * 60,
+        useNetServer: false,
+        netServerTimeoutInMS: 5 * 60 * 1000,
+        socketAddress: {
+            port: 1212,
+            host: 'localhost'
+        }
+    }, options || {});
+
+    if (options.useNetServer) {
+        return useNetServer(options);
+    }
+
+    return usePortPolling(options);
+};