Add graceful shutdown of API when receiving SIGTERM

docker-compose.yaml

@@ -79,12 +79,13 @@ services:
       - ./node-packages:/app/node-packages:delegated
   api:
     image: ${IMAGE_REPO:-lagoon}/api
-    command: yarn run dev
+    command: ./node_modules/.bin/tsc-watch --build --incremental --onSuccess "node -r dotenv-extended/config dist/index"
     volumes:
       - ./services/api/src:/app/services/api/src
       - ./node-packages:/app/node-packages:delegated
       - /app/node-packages/commons/dist
     environment:
+      - NODE_ENV=development
       - CI=${CI:-true}
       - REGISTRY=harbor.172.17.0.1.nip.io:18080 # Docker network bridge and forwarded port for harbor-nginx
     depends_on:

services/api/Dockerfile

@@ -43,4 +43,4 @@ COPY wait-for-mariadb.sh /lagoon/entrypoints/99-wait-for-mariadb.sh
 
 RUN yarn build
 
-CMD ["yarn", "start"]
+CMD ["node", "-r", "dotenv-extended/config", "dist/index"]

services/api/package.json

@@ -9,8 +9,6 @@
     "test": "jest --forceExit --detectOpenHandles --maxWorkers=10",
     "test:watch": "jest --forceExit --detectOpenHandles --maxWorkers=10 --watch",
     "build": "tsc --build",
-    "start": "node -r dotenv-extended/config dist/index",
-    "dev": "NODE_ENV=development nodemon",
     "sync:gitlab:users": "node dist/gitlab-sync/users",
     "sync:gitlab:groups": "node dist/gitlab-sync/groups",
     "sync:gitlab:projects": "node dist/gitlab-sync/projects",
@@ -19,18 +17,6 @@
     "sync:bitbucket:repo-permissions": "node dist/bitbucket-sync/repo-permissions",
     "sync:harbor:projects": "node dist/migrations/2-harbor/harborSync.js"
   },
-  "nodemonConfig": {
-    "ignore": [
-      "*.test.js",
-      "../../node-packages/commons/dist/"
-    ],
-    "watch": [
-      "src",
-      "../../node-packages/"
-    ],
-    "ext": "js,ts,json",
-    "exec": "yarn build --incremental && yarn start --inspect=0.0.0.0:9229"
-  },
   "keywords": [],
   "author": "amazee.io <hello@amazee.io> (http://www.amazee.io)",
   "license": "MIT",
@@ -81,10 +67,10 @@
     "axios": "^0.19.0",
     "faker": "^4.1.0",
     "jest": "^24.9.0",
-    "nodemon": "^1.12.1",
     "prettier": "^1.14.2",
     "prettier-eslint-cli": "^4.7.1",
     "ts-jest": "^24.0.2",
+    "tsc-watch": "^4.4.0",
     "typescript": "^3.4.4"
   }
 }

services/api/src/app.ts

@@ -0,0 +1,36 @@
+import express from 'express';
+import morgan from 'morgan';
+import compression from 'compression';
+import cors from 'cors';
+import { json } from 'body-parser';
+import { logger } from './loggers/logger';
+import { createRouter } from './routes';
+import { authMiddleware } from './authMiddleware';
+import apolloServer from './apolloServer';
+
+export const app = express();
+
+// Use compression (gzip) for responses.
+app.use(compression());
+
+// Automatically decode json.
+app.use(json());
+
+// Add custom configured logger (morgan through winston).
+app.use(
+  morgan('combined', {
+    stream: {
+      write: message => logger.info(message)
+    }
+  })
+);
+
+// TODO: Restrict requests to lagoon domains?
+app.use(cors());
+
+app.use(authMiddleware);
+
+// Add routes.
+app.use('/', createRouter());
+
+apolloServer.applyMiddleware({ app });

services/api/src/index.ts

@@ -0,0 +1,57 @@
+import 'newrelic';
+import { Server } from 'http';
+import { promisify } from 'util';
+import { initSendToLagoonLogs } from '@lagoon/commons/dist/logs';
+import { initSendToLagoonTasks } from '@lagoon/commons/dist/tasks';
+import { waitForKeycloak } from './util/waitForKeycloak';
+import { envHasConfig } from './util/config';
+import { logger } from './loggers/logger';
+import { createServer } from './server';
+
+initSendToLagoonLogs();
+initSendToLagoonTasks();
+
+const makeGracefulShutdown = (server: Server) => {
+  return async (signal: NodeJS.Signals) => {
+    logger.debug(`${signal}: API Shutting Down`);
+
+    logger.debug('Closing sqlClientPool');
+    const { sqlClientPool } = await import('./clients/sqlClient');
+    await sqlClientPool.end();
+
+    process.kill(process.pid, signal);
+  };
+};
+
+(async () => {
+  await waitForKeycloak();
+
+  logger.debug('Starting to boot the application.');
+
+  try {
+    if (!envHasConfig('JWTSECRET')) {
+      throw new Error(
+        'Required environment variable JWTSECRET is undefined or null!'
+      );
+    }
+
+    if (!envHasConfig('JWTAUDIENCE')) {
+      throw new Error(
+        'Required environment variable JWTAUDIENCE is undefined or null!'
+      );
+    }
+
+    const server = await createServer();
+    const gracefulShutdown = makeGracefulShutdown(server);
+
+    // Shutdown on tsc-watch restart, docker-compose restart/kill, and k8s pod kills
+    process.once('SIGTERM', gracefulShutdown);
+    // Shutdown on ctrl-c
+    process.once('SIGINT', gracefulShutdown);
+
+    logger.debug('Finished booting the application.');
+  } catch (e) {
+    logger.error('Error occurred while starting the application');
+    logger.error(e.stack);
+  }
+})();

services/api/src/server.ts

@@ -0,0 +1,26 @@
+import http from 'http';
+import util from 'util';
+import { logger } from './loggers/logger';
+import { toNumber } from './util/func';
+import { getConfigFromEnv } from './util/config';
+import { app } from './app';
+import apolloServer from './apolloServer';
+
+export const createServer = async () => {
+  logger.debug('Starting to boot the server.');
+
+  const port = toNumber(getConfigFromEnv('PORT', '3000'));
+  const server = http.createServer(app);
+  server.setTimeout(900000); // higher Server timeout: 15min instead of default 2min
+
+  apolloServer.installSubscriptionHandlers(server);
+
+  const listen = util.promisify(server.listen).bind(server);
+  await listen(port);
+
+  logger.debug(
+    `Finished booting the server. The server is reachable at port ${port.toString()}.`
+  );
+
+  return server;
+};

services/api/src/util/waitForKeycloak.js → services/api/src/util/waitForKeycloak.ts

@@ -1,7 +1,7 @@
 import { logger } from '../loggers/logger';
-const { getKeycloakAdminClient } = require('../clients/keycloak-admin');
+import { getKeycloakAdminClient } from '../clients/keycloak-admin';
 
-async function waitForKeycloak() {
+export const waitForKeycloak = async () => {
   let keycloakReady = false;
   let keycloakAdminClient;
 
@@ -13,7 +13,9 @@ async function waitForKeycloak() {
         throw new Error('The "lagoon" realm has not been created yet.');
       }
 
-      const clients = await keycloakAdminClient.clients.find({ clientId: 'api' });
+      const clients = await keycloakAdminClient.clients.find({
+        clientId: 'api'
+      });
       if (!clients.length) {
         throw new Error('The "api" client has not been created yet.');
       }
@@ -30,6 +32,4 @@ async function waitForKeycloak() {
   }
 
   logger.debug('Connected to Keycloak');
-}
-
-module.exports = waitForKeycloak;
+};