Update onHealthCheck. Allow longer for liveness to respond

razee-io · Mar 23, 2021 · 5cb7475 · 5cb7475
1 parent 3f42fe5
commit 5cb7475
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 25 deletions.
diff --git a/app/apollo/index.js b/app/apollo/index.js
@@ -92,7 +92,7 @@ const buildCommonApolloContext = async ({ models, req, res, connection, logger }
     context = { apiKey: apiKey, req: upgradeReq, req_id: upgradeReq ? upgradeReq.id : undefined, userToken, recoveryHintsMap, orgId, ...context };
   } else if (req) {
     context = { req, req_id: req.id, recoveryHintsMap, ...context };
-  } 
+  }
   return context;
 };
 
@@ -112,6 +112,9 @@ const loadCustomPlugins =  () => {
   return [];
 };
 
+var SIGTERM = false;
+process.on('SIGTERM', () => SIGTERM = true);
+
 const createApolloServer = () => {
   const customPlugins = loadCustomPlugins();
   if (process.env.GRAPHQL_ENABLE_TRACING === 'true') {
@@ -165,15 +168,15 @@ const createApolloServer = () => {
 
         logger.trace({ req_id, connectionParams, context }, 'subscriptions:onConnect');
         const me = await models.User.getMeFromConnectionParams( connectionParams, {req_id, models, logger, ...context},);
-        
+
         logger.debug({ me }, 'subscriptions:onConnect upgradeReq getMe');
         if (me === undefined) {
           throw Error(
             'Can not find the session for this subscription request.',
           );
         }
-        
-        // add original upgrade request to the context 
+
+        // add original upgrade request to the context
         return { me, upgradeReq: webSocket.upgradeReq, logger, orgKey, orgId };
       },
       onDisconnect: (webSocket, context) => {
@@ -212,14 +215,14 @@ const apollo = async (options = {}) => {
     server.applyMiddleware({
       app,
       path: GRAPHQL_PATH,
-      onHealthCheck: () => {
-        return new Promise((resolve, reject) => {
-          if (pubSub.enabled) {
-            resolve();
-          } else {
-            reject();
-          }
-        });
+      onHealthCheck: async () => {
+        if (SIGTERM) {
+          throw 'SIGTERM received. Not accepting additional requests';
+        } else if (!pubSub.enabled){
+          throw '!pubSub.enabled';
+        } else {
+          return 200;
+        }
       }
     });
 
@@ -239,7 +242,7 @@ const apollo = async (options = {}) => {
         port = options.graphql_port;
       }
       httpServer.listen({ port });
-    } 
+    }
     return { db, server, httpServer, stop};
   } catch (err) {
     logger.error(err, 'Apollo api error');

diff --git a/app/routes/kube/kube.js b/app/routes/kube/kube.js
@@ -24,7 +24,7 @@ const logger = bunyan.createLogger(getBunyanConfig('kube/liveness'));
 const timeInterval = 300000; //5 mintues
 
 // /kube/liveness
-const kube = router.get('/liveness', asyncHandler(async(req, res) => {
+router.get('/liveness', asyncHandler(async(req, res) => {
   // does a db call to make sure we didnt disconnect
   try {
     await require('../../apollo/models').models.Organization.findOne({});
@@ -40,7 +40,6 @@ const kube = router.get('/liveness', asyncHandler(async(req, res) => {
     logger.error('Razeedash Api is down due to Redis pubsub connection issue, please check logs.');
     return res.sendStatus(503);
   }
-
   if (pubSub.lastPubSubMessage !== null && Date.now()- pubSub.lastPubSubMessage.time > timeInterval) {
     // check if the most recent message received is within ${timeInterval/60000} minitue
     logger.error(`Razeedash Api is down, haven't received any published messages within ${timeInterval/60000} minitue, please check logs.`);
@@ -49,4 +48,4 @@ const kube = router.get('/liveness', asyncHandler(async(req, res) => {
   return res.sendStatus(200);
 }));
 
-module.exports = kube;
+module.exports = router;
diff --git a/kubernetes/razeedash-api/resource.yaml b/kubernetes/razeedash-api/resource.yaml
@@ -105,21 +105,22 @@ items:
                   path: /api/kube/liveness
                   port: 3333
                 initialDelaySeconds: 5
-                periodSeconds: 20
-                timeoutSeconds: 2
+                periodSeconds: 30
+                timeoutSeconds: 10
               readinessProbe:
-                exec:
-                  command: ["npm", "run", "wait-mongo"]
+                httpGet:
+                  path: /.well-known/apollo/server-health
+                  port: 3333
                 initialDelaySeconds: 5
-                periodSeconds: 15
+                periodSeconds: 30
                 timeoutSeconds: 10
               resources:
                 requests:
-                  cpu: 100m
-                  memory: 80Mi
-                limits:
                   cpu: 500m
-                  memory: 256Mi
+                  memory: 512Mi
+                limits:
+                  cpu: 2000m
+                  memory: 1Gi
               volumeMounts:
                 - name: add-cluster-webhook-headers-vol
                   mountPath: /var/run/secrets/razeeio/razeedash-api/add-cluster-webhook-headers