Limit number of restarts. Validate container logs for better error messages.

package.json

@@ -30,6 +30,7 @@
     "babel-polyfill": "6.26.0",
     "buntstift": "1.5.1",
     "certificate-details": "0.3.1",
+    "combined-stream": "1.0.6",
     "command-line-args": "5.0.2",
     "command-line-commands": "2.0.1",
     "command-line-usage": "5.0.5",
@@ -66,7 +67,7 @@
   "devDependencies": {
     "assertthat": "1.0.0",
     "measure-time": "3.1.1",
-    "roboter": "1.0.5"
+    "roboter": "1.0.6"
   },
   "scripts": {
     "test-stories": "node test/stories/index.js"

src/cli/commands/start.js

@@ -105,17 +105,32 @@ const init = {
     } catch (ex) {
       stopWaiting();
 
-      if (ex.code === 'ECODEMALFORMED') {
-        const formatter = eslint.CLIEngine.getFormatter();
-
-        const formattedResult = formatter(ex.cause.results);
-        const output = formattedResult.
-          split('\n').
-          slice(0, -2).
-          join('\n');
-
-        buntstift.info(output);
-        buntstift.info(ex.message);
+      switch (ex.code) {
+        case 'ECODEMALFORMED': {
+          const formatter = eslint.CLIEngine.getFormatter();
+
+          const formattedResult = formatter(ex.cause.results);
+          const output = formattedResult.
+            split('\n').
+            slice(0, -2).
+            join('\n');
+
+          buntstift.info(output);
+          buntstift.info(ex.message);
+          break;
+        }
+        case 'ERUNTIMEERROR': {
+          if (ex.orginalError) {
+            buntstift.newLine();
+            buntstift.info(ex.orginalError.stack);
+            buntstift.newLine();
+          }
+
+          buntstift.info('Application code caused runtime error.');
+          break;
+        }
+        default:
+          break;
       }
 
       buntstift.error('Failed to start the application.');

src/configuration/latest/broker/container.js

@@ -85,7 +85,7 @@ const container = function (options) {
       443: selectedEnvironment.api.address.port,
       3333: selectedEnvironment.api.address.port + 9
     },
-    restart: 'always',
+    restart: 'on-failure:3',
     volumesFrom: [
       `${configuration.application}-node-modules`
     ]

src/configuration/latest/core/container.js

@@ -70,7 +70,7 @@ const container = function (options) {
     ports: {
       3333: selectedEnvironment.api.address.port + 10
     },
-    restart: 'always',
+    restart: 'on-failure:3',
     volumesFrom: [
       `${configuration.application}-node-modules`
     ]

src/configuration/latest/depot-file/container.js

@@ -58,7 +58,7 @@ const container = function (options) {
       443: selectedEnvironment.api.address.port + 1,
       3333: selectedEnvironment.api.address.port + 12
     },
-    restart: 'always',
+    restart: 'on-failure:3',
     volumes: [
       '/blobs'
     ]

src/configuration/latest/flows/container.js

@@ -68,7 +68,7 @@ const container = function (options) {
     ports: {
       3333: selectedEnvironment.api.address.port + 11
     },
-    restart: 'always',
+    restart: 'on-failure:3',
     volumesFrom: [
       `${configuration.application}-node-modules`
     ]

src/configuration/latest/mongodb/container.js

@@ -52,7 +52,7 @@ const container = function (options) {
     ports: {
       27017: selectedEnvironment.api.address.port + 2
     },
-    restart: 'always'
+    restart: 'on-failure:3'
   };
 
   if (persistData) {

src/configuration/latest/postgres/container.js

@@ -52,7 +52,7 @@ const container = function (options) {
     ports: {
       5432: selectedEnvironment.api.address.port + 3
     },
-    restart: 'always'
+    restart: 'on-failure:3'
   };
 
   if (persistData) {

src/configuration/latest/rabbitmq/container.js

@@ -52,7 +52,7 @@ const container = function (options) {
       5672: selectedEnvironment.api.address.port + 4,
       15672: selectedEnvironment.api.address.port + 5
     },
-    restart: 'always'
+    restart: 'on-failure:3'
   };
 
   if (persistData) {

src/docker/logs.js

@@ -1,5 +1,7 @@
 'use strict';
 
+const combinedStream = require('combined-stream');
+
 const getEnvironmentVariables = require('./getEnvironmentVariables'),
       shell = require('../shell');
 
@@ -20,7 +22,7 @@ const logs = async function (options) {
     throw new Error('Follow is missing.');
   }
 
-  const { configuration, containers, env, follow } = options;
+  const { configuration, containers, env, follow, passThrough } = options;
 
   const environmentVariables = await getEnvironmentVariables({ configuration, env });
 
@@ -35,9 +37,9 @@ const logs = async function (options) {
       args.push('--follow');
     }
 
-    const child = shell.spawn('docker', args, { env: environmentVariables, stdio: 'inherit' });
+    const child = shell.spawn('docker', args, { env: environmentVariables, stdio: 'pipe' });
 
-    child.on('close', code => {
+    child.once('close', code => {
       if (code !== 0) {
         childProcesses.forEach(process => {
           process.kill();
@@ -47,12 +49,27 @@ const logs = async function (options) {
         process.exit(1);
         /* eslint-enable no-process-exit */
       }
-
-      resolve();
     });
 
     childProcesses.push(child);
+
+    resolve();
   })));
+
+  const multiStream = combinedStream.create();
+
+  childProcesses.
+    map(child => child.stdout).
+    forEach(stream => multiStream.append(stream));
+
+  const outputStream = passThrough || process.stdout;
+
+  await new Promise((resolve, reject) => {
+    multiStream.once('error', reject);
+    multiStream.once('end', resolve);
+
+    multiStream.pipe(outputStream);
+  });
 };
 
 module.exports = logs;

src/errors.js

@@ -30,8 +30,9 @@ const errors = defekt([
   'PortsNotAvailable',
   'ProtocolInvalid',
   'RequestFailed',
-  'RuntimeInUse',
   'RuntimeAlreadyInstalled',
+  'RuntimeError',
+  'RuntimeInUse',
   'RuntimeNotInstalled',
   'UnknownError',
   'UrlMalformed',

src/wolkenkit/commands/reload/index.js

@@ -56,7 +56,6 @@ const reload = async function (options, progress = noop) {
   }
 
   const debug = existingContainers[0].labels['wolkenkit-debug'] === 'true',
-        host = existingContainers[0].labels['wolkenkit-api-host'],
         persistData = existingContainers[0].labels['wolkenkit-persist-data'] === 'true',
         port = Number(existingContainers[0].labels['wolkenkit-api-port']),
         sharedKey = existingContainers[0].labels['wolkenkit-shared-key'];
@@ -78,9 +77,8 @@ const reload = async function (options, progress = noop) {
   await startContainers({ configuration, env, port, sharedKey, persistData, debug }, progress);
 
   progress({ message: `Using ${sharedKey} as shared key.`, type: 'info' });
-  progress({ message: `Waiting for https://${host}:${port}/v1/ping to reply...`, type: 'info' });
 
-  await shared.waitForApplication({ configuration, env }, progress);
+  await shared.waitForApplicationAndValidateLogs({ configuration, env }, progress);
 
   if (debug) {
     await shared.attachDebugger({ configuration, env, sharedKey, persistData, debug }, progress);

src/wolkenkit/commands/shared/validateLogs.js

@@ -0,0 +1,104 @@
+'use strict';
+
+const EventEmitter = require('events'),
+      { promisify } = require('util');
+
+const { Parser } = require('newline-json');
+
+const docker = require('../../../docker'),
+      errors = require('../../../errors');
+
+const sleep = promisify(setTimeout);
+
+const validateLogs = async function (options, progress) {
+  if (!options) {
+    throw new Error('Options are missing');
+  }
+  if (!options.configuration) {
+    throw new Error('Configuration is missing.');
+  }
+  if (!options.env) {
+    throw new Error('Environment is missing.');
+  }
+  if (!progress) {
+    throw new Error('Progress is missing.');
+  }
+
+  const { configuration, env } = options;
+
+  const containers = await docker.getContainers({
+    configuration,
+    env,
+    where: { label: { 'wolkenkit-application': configuration.application, 'wolkenkit-type': 'application' }}
+  });
+
+  progress({ message: 'Validating container logs...', type: 'info' });
+
+  const validate = new EventEmitter();
+
+  let isStopped = false;
+
+  validate.once('stop', () => {
+    isStopped = true;
+  });
+
+  (async () => {
+    while (!isStopped) {
+      try {
+        await new Promise(async (resolve, reject) => {
+          const passThrough = new Parser();
+
+          let unsubscribe;
+
+          const onData = logMessage => {
+            if (logMessage.level === 'fatal') {
+              let orginalError = null;
+
+              if (logMessage.metadata) {
+                orginalError = logMessage.metadata.err || logMessage.metadata.ex;
+              }
+
+              const runtimeError = new errors.RuntimeError('Fatal runtime error happened.');
+
+              runtimeError.orginalError = orginalError;
+              runtimeError.logMessage = logMessage;
+
+              unsubscribe();
+
+              return reject(runtimeError);
+            }
+          };
+
+          unsubscribe = () => {
+            passThrough.removeListener('data', onData);
+          };
+
+          passThrough.on('data', onData);
+
+          passThrough.once('end', () => {
+            unsubscribe();
+            resolve();
+          });
+
+          try {
+            await docker.logs({ configuration, containers, env, follow: false, passThrough });
+          } catch (ex) {
+            reject(ex);
+          }
+        });
+      } catch (ex) {
+        isStopped = true;
+
+        validate.emit('error', ex);
+      }
+
+      // We don't want to collect the logs as often as possible.
+      // Because this can cause to performance issues, hence the sleep timeout.
+      await sleep(250);
+    }
+  })();
+
+  return validate;
+};
+
+module.exports = validateLogs;

src/wolkenkit/commands/shared/waitForApplicationAndValidateLogs.js

@@ -0,0 +1,41 @@
+'use strict';
+
+const validateLogs = require('./validateLogs'),
+      waitForApplication = require('./waitForApplication');
+
+const waitForApplicationAndValidateLogs = async function (options, progress) {
+  if (!options) {
+    throw new Error('Options are missing.');
+  }
+  if (!options.configuration) {
+    throw new Error('Configuration is missing.');
+  }
+  if (!options.env) {
+    throw new Error('Environment is missing.');
+  }
+  if (!progress) {
+    throw new Error('Progress is missing.');
+  }
+
+  const { configuration, env } = options;
+
+  await new Promise(async (resolve, reject) => {
+    let validate;
+
+    try {
+      validate = await validateLogs({ configuration, env }, progress);
+
+      validate.once('error', reject);
+
+      await waitForApplication({ configuration, env }, progress);
+    } catch (ex) {
+      return reject(ex);
+    } finally {
+      validate.emit('stop');
+    }
+
+    resolve();
+  });
+};
+
+module.exports = waitForApplicationAndValidateLogs;

src/wolkenkit/commands/start/cli/index.js

@@ -7,9 +7,11 @@ const docker = require('../../../../docker'),
       generateSharedKey = require('./generateSharedKey'),
       health = require('../../health'),
       install = require('../../install'),
+      noop = require('../../../../noop'),
       runtimes = require('../../../runtimes'),
       shared = require('../../shared'),
       startContainers = require('./startContainers'),
+      stop = require('../../stop'),
       verifyThatPortsAreAvailable = require('./verifyThatPortsAreAvailable');
 
 const cli = async function (options, progress) {
@@ -94,7 +96,19 @@ const cli = async function (options, progress) {
 
   progress({ message: `Using ${sharedKey} as shared key.`, type: 'info' });
 
-  await shared.waitForApplication({ configuration, env }, progress);
+  try {
+    await shared.waitForApplicationAndValidateLogs({ configuration, env }, progress);
+  } catch (ex) {
+    switch (ex.code) {
+      case 'ERUNTIMEERROR':
+        await stop({ directory, dangerouslyDestroyData: false, env, configuration }, noop);
+        break;
+      default:
+        break;
+    }
+
+    throw ex;
+  }
 
   if (debug) {
     await shared.attachDebugger({ configuration, env, sharedKey, persistData, debug }, progress);