Merge pull request #808 from razee-io/readiness

Update readiness to rely on apollo health, extended with SIGTERM check
razee-io · Mar 23, 2021 · a210c28 · a210c28
2 parents efdbbbf + 92f9e2c
commit a210c28
Show file tree

Hide file tree

Showing 5 changed files with 35 additions and 23 deletions.
diff --git a/app/apollo/index.js b/app/apollo/index.js
@@ -112,6 +112,9 @@ const loadCustomPlugins =  () => {
   return [];
 };
 
+var SIGTERM = false;
+process.on('SIGTERM', () => SIGTERM = true);
+
 const createApolloServer = () => {
   const customPlugins = loadCustomPlugins();
   if (process.env.GRAPHQL_ENABLE_TRACING === 'true') {
@@ -212,14 +215,14 @@ const apollo = async (options = {}) => {
     server.applyMiddleware({
       app,
       path: GRAPHQL_PATH,
-      onHealthCheck: () => {
-        return new Promise((resolve, reject) => {
-          if (pubSub.enabled) {
-            resolve();
-          } else {
-            reject();
-          }
-        });
+      onHealthCheck: async () => {
+        if (SIGTERM) {
+          throw 'SIGTERM received. Not accepting additional requests';
+        } else if (!pubSub.enabled){
+          throw '!pubSub.enabled';
+        } else {
+          return 200;
+        }
       }
     });
 

diff --git a/app/apollo/utils/bunyan.js b/app/apollo/utils/bunyan.js
@@ -18,7 +18,7 @@ const responseCodeMapper = (status, err, meta) => {
   if (meta['req-headers'] && meta['req-headers']['authorization']) {
     meta['req-headers']['authorization'] = 'Bearer [HIDDEN]';
     meta['req-headers']['x-auth-refresh-token'] = 'Bearer [HIDDEN]';
-  } 
+  }
   if (meta.method === 'OPTIONS' && status === 204) {
     // skip OPTION request 204 response
     return 'trace';
@@ -37,16 +37,21 @@ const responseCodeMapper = (status, err, meta) => {
 };
 
 const getBunyanConfig = (route) => {
-  return {
+  const result = {
     name: route,
     parseUA: false,
-    excludes: ['referer', 'url', 'short-body', 'user-agent', 'req', 'res'],
+    excludes: ['referer', 'body', 'short-body'],
     levelFn: responseCodeMapper,
+    obfuscate: ['req.headers.razee-org-key'],
+    genReqId: function (req) {
+      return req.request_id;
+    },
     streams: [{
       level: process.env.LOG_LEVEL || 'info',
       stream: process.stdout
-    }]      
+    }]
   };
+  return result;
 };
 
 module.exports = { getBunyanConfig };
diff --git a/app/routes/kube/kube.js b/app/routes/kube/kube.js
@@ -24,7 +24,7 @@ const logger = bunyan.createLogger(getBunyanConfig('razeedash-api/kube/liveness'
 const timeInterval = 300000; //5 mintues
 
 // /kube/liveness
-const kube = router.get('/liveness', asyncHandler(async(req, res) => {
+router.get('/liveness', asyncHandler(async(req, res) => {
   // does a db call to make sure we didnt disconnect
   try {
     await require('../../apollo/models').models.Organization.findOne({});
@@ -49,4 +49,4 @@ const kube = router.get('/liveness', asyncHandler(async(req, res) => {
   return res.sendStatus(200);
 }));
 
-module.exports = kube;
+module.exports = router;
diff --git a/app/utils/bunyan.js b/app/utils/bunyan.js
@@ -33,6 +33,9 @@ const getBunyanConfig = (route) => {
     excludes: ['referer', 'body', 'short-body'],
     levelFn: responseCodeMapper,
     obfuscate: ['req.headers.razee-org-key'],
+    genReqId: function (req) {
+      return req.request_id;
+    },
     streams: [{
       level: process.env.LOG_LEVEL || 'info',
       stream: process.stdout

diff --git a/kubernetes/razeedash-api/resource.yaml b/kubernetes/razeedash-api/resource.yaml
@@ -105,21 +105,22 @@ items:
                   path: /api/kube/liveness
                   port: 3333
                 initialDelaySeconds: 5
-                periodSeconds: 20
-                timeoutSeconds: 2
+                periodSeconds: 30
+                timeoutSeconds: 10
               readinessProbe:
-                exec:
-                  command: ["npm", "run", "wait-mongo"]
+                httpGet:
+                  path: /.well-known/apollo/server-health
+                  port: 3333
                 initialDelaySeconds: 5
-                periodSeconds: 15
+                periodSeconds: 30
                 timeoutSeconds: 10
               resources:
                 requests:
-                  cpu: 100m
-                  memory: 80Mi
-                limits:
                   cpu: 500m
-                  memory: 256Mi
+                  memory: 512Mi
+                limits:
+                  cpu: 2000m
+                  memory: 1Gi
               volumeMounts:
                 - name: add-cluster-webhook-headers-vol
                   mountPath: /var/run/secrets/razeeio/razeedash-api/add-cluster-webhook-headers