fix(core): small changes to diagnostics (#11563)

packages/bp/src/core/app/botpress.ts

@@ -203,6 +203,7 @@ export class Botpress {
       } else {
         const { endpoint } = config.nluServer
         this.logger.info(`NLU server manually handled at: ${endpoint}`)
+        process.NLU_ENDPOINT = endpoint
       }
 
       return

packages/bp/src/core/distributed/redis.ts

@@ -1,3 +1,4 @@
+import { printRow } from 'diag/utils'
 import RedisIo, { Cluster, ClusterNode, ClusterOptions, Redis, RedisOptions } from 'ioredis'
 import _ from 'lodash'
 
@@ -82,8 +83,9 @@ const parseLine = (client: string): ClientEntry => {
 }
 
 export const getClientsList = async (redis: Redis): Promise<ClientEntry[]> => {
+  let clients
   try {
-    const clients = await redis.client('list')
+    clients = await redis.client('list')
 
     // We don't want to get clients which issues commands
     const subscribers = clients
@@ -93,7 +95,8 @@ export const getClientsList = async (redis: Redis): Promise<ClientEntry[]> => {
 
     return _.uniqBy(subscribers, x => x.parsed.name)
   } catch (err) {
-    console.error('error parsing clients', err)
+    printRow('Error parsing redis clients', err)
+    printRow('Clients returned:', clients)
   }
 
   return []

packages/bp/src/diag/index.ts

@@ -5,6 +5,8 @@ import '../sdk/rewire'
 
 import { BotConfig } from 'botpress/sdk'
 
+import chalk from 'chalk'
+import { makeNLUPassword } from 'common/nlu-token'
 import { Workspace } from 'common/typings'
 import { BotpressApp, createApp } from 'core/app/core-loader'
 import { getClientsList, getOrCreate as redisFactory, makeRedisKey } from 'core/distributed'
@@ -16,6 +18,7 @@ import { nanoid } from 'nanoid'
 import os from 'os'
 import path from 'path'
 import stripAnsi from 'strip-ansi'
+import url from 'url'
 import yn from 'yn'
 import { startMonitor } from './monitor'
 import {
@@ -24,6 +27,7 @@ import {
   printObject,
   printRow,
   printSub,
+  queryWebsite,
   testWebsiteAccess,
   testWriteAccess,
   wrapMethodCall
@@ -37,6 +41,14 @@ interface Options {
   monitor?: boolean
 }
 
+interface BpProcesses {
+  label: string
+  endpoint?: string
+  port?: number
+  page: string
+  headers?: any
+}
+
 export const OBFUSCATED = '***obfuscated***'
 export const SECRET_KEYS = ['secret', 'pw', 'password', 'token', 'key', 'cert']
 export const ENV_VARS = [
@@ -142,41 +154,49 @@ const testConnectivity = async () => {
     await app.ghost.initialize(useDbDriver)
   })
 
-  if (process.env.CLUSTER_ENABLED && redisFactory) {
-    await wrapMethodCall('Connecting to Redis', async () => {
-      redisClient = redisFactory('commands')
+  if (!process.env.CLUSTER_ENABLED || !redisFactory) {
+    return
+  }
 
-      if ((await redisClient.ping().timeout(3000)) !== 'PONG') {
-        throw new Error("The server didn't answer our ping request after 3 seconds")
-      }
-    })
+  await wrapMethodCall('Connecting to Redis', async () => {
+    redisClient = redisFactory('commands')
 
-    await wrapMethodCall('Basic test of Redis', async () => {
-      const key = makeRedisKey(REDIS_TEST_KEY)
-      await redisClient.set(key, REDIS_TEST_VALUE)
-      const fetchValue = await redisClient.get(key)
-      await redisClient.del(key)
+    if ((await redisClient.ping().timeout(3000)) !== 'PONG') {
+      throw new Error("The server didn't answer our ping request after 3 seconds")
+    }
+  })
 
-      if (fetchValue !== REDIS_TEST_VALUE) {
-        throw new Error('Could not complete a basic operation on Redis')
-      }
-    })
+  await wrapMethodCall('Basic test of Redis', async () => {
+    const key = makeRedisKey(REDIS_TEST_KEY)
+    await redisClient.set(key, REDIS_TEST_VALUE)
+    const fetchValue = await redisClient.get(key)
+    await redisClient.del(key)
+
+    if (fetchValue !== REDIS_TEST_VALUE) {
+      throw new Error('Could not complete a basic operation on Redis')
+    }
+  })
 
-    try {
-      // @ts-ignore typing missing for that method
-      const reply = await redisClient.pubsub(['NUMSUB', makeRedisKey('job_done')])
-      process.env.BP_REDIS_SCOPE && printRow('Redis using scope', process.env.BP_REDIS_SCOPE)
-      printRow('Botpress nodes listening on Redis', reply[1])
-    } catch (err) {}
+  try {
+    // @ts-ignore typing missing for that method
+    const reply = await redisClient.pubsub(['NUMSUB', makeRedisKey('job_done')])
+    process.env.BP_REDIS_SCOPE && printRow('Redis using scope', process.env.BP_REDIS_SCOPE)
+    printRow('Botpress nodes listening on Redis', reply[1])
+  } catch (err) {}
 
-    try {
-      for (const client of await getClientsList(redisClient)) {
-        printRow(`- Client ${client.parsed.name}`, `Uptime: ${client.parsed.age}s`)
-      }
-    } catch (err) {
-      printRow('- Error getting clients list', err)
+  try {
+    for (const client of await getClientsList(redisClient)) {
+      printRow(`- Client ${client.parsed.name}`, `Uptime: ${client.parsed.age}s`)
     }
+  } catch (err) {
+    printRow('- Error getting clients list', err)
   }
+
+  try {
+    // @ts-ignore typing missing for that method
+    const reply = await redisClient.pubsub(['NUMSUB', makeRedisKey('cache_status')])
+    printRow('Messaging nodes listening on Redis', reply[1])
+  } catch (err) {}
 }
 
 const testNetworkConnections = async () => {
@@ -213,22 +233,75 @@ const testNetworkConnections = async () => {
   await Promise.map(hosts, host => testWebsiteAccess(host.label, host.url))
 }
 
-const printDatabaseTables = async () => {
-  let tables
-  if (app.database.knex.isLite) {
-    tables = await app.database.knex
-      .raw("SELECT name FROM sqlite_master WHERE type='table'")
-      .then(res => res.map(x => x.name))
-  } else {
-    tables = await app.database
-      .knex('pg_catalog.pg_tables')
-      .select('tablename')
-      .where({ schemaname: 'public' })
-      .then(res => res.map(x => x.tablename))
+const testServiceConnections = async () => {
+  // This test only works when the server is running
+  if (!process.STUDIO_PORT) {
+    return
   }
 
+  printHeader('Connection to Services')
+
+  const hosts = ['localhost', '127.0.0.1', os.hostname(), new url.URL(process.EXTERNAL_URL).hostname]
+
+  const processes: BpProcesses[] = [{ label: 'Studio', port: process.STUDIO_PORT, page: 'status' }]
+
+  if (process.MESSAGING_PORT || process.core_env.MESSAGING_ENDPOINT) {
+    processes.push({
+      label: 'Messaging',
+      port: process.MESSAGING_PORT,
+      endpoint: process.core_env.MESSAGING_ENDPOINT,
+      page: 'status'
+    })
+  }
+
+  if (process.NLU_PORT || process.NLU_ENDPOINT) {
+    processes.push({
+      label: 'NLU',
+      port: process.NLU_PORT,
+      endpoint: process.NLU_ENDPOINT,
+      page: 'info',
+      headers: { Authorization: `Bearer ${makeNLUPassword()}` }
+    })
+  }
+
+  for (const { label, port, endpoint, page, headers } of processes) {
+    const urls = endpoint
+      ? [`${endpoint}/${page}`]
+      : [...hosts.map(x => `${x.startsWith('http') ? x : `http://${x}`}:${port}/${page}`)]
+
+    const results = await Promise.mapSeries(urls, url => queryWebsite(url, headers))
+    const parsed = results
+      .map(x => {
+        const host = new url.URL(x.url).hostname
+
+        return x.success ? chalk.green(`${host} (ok)`) : chalk.red(`${host} ${x.url} (${x.message})`)
+      })
+      .join(', ')
+
+    printRow(`${label} ${endpoint ? `(${endpoint})` : `(port ${port})`}`, parsed)
+  }
+}
+
+const printDatabaseTables = async () => {
   printHeader('Database Tables')
-  print(tables.sort().join(', '))
+
+  let tables
+  try {
+    if (app.database.knex.isLite) {
+      tables = await app.database.knex
+        .raw("SELECT name FROM sqlite_master WHERE type='table'")
+        .then(res => res.map(x => x.name))
+    } else {
+      tables = await app.database
+        .knex('pg_catalog.pg_tables')
+        .select('tablename')
+        .where({ schemaname: 'public' })
+        .then(res => res.map(x => x.tablename))
+    }
+    print(tables.sort().join(', '))
+  } catch (err) {
+    printRow('Error while loading tables', err)
+  }
 }
 
 const listEnvironmentVariables = () => {
@@ -305,6 +378,7 @@ export default async function(options: Options) {
   } catch (err) {}
 
   await testNetworkConnections()
+  await testServiceConnections()
 
   if (options.config || yn(process.env.BP_DIAG_CONFIG)) {
     await printConfig()

packages/bp/src/diag/utils.ts

@@ -75,6 +75,31 @@ export const testWebsiteAccess = async (label: string, url: string) => {
   print(` - ${url} (${ip})\n`)
 }
 
+export const queryWebsite = async (url: string, headers?: any) => {
+  const start = Date.now()
+  try {
+    await axios.get(url, { headers })
+    return { url, success: true, delay: Date.now() - start }
+  } catch (err) {
+    return { url, success: false, delay: Date.now() - start, message: err.message, status: err.response?.status }
+  }
+}
+
+export const testProcessAccess = async (label: string, url: string) => {
+  const start = Date.now()
+  let ip
+  try {
+    ip = await dnsLookup(new URL(url).hostname)
+    await axios.get(url)
+
+    printRow(label, `${chalk.green('success')} (${Date.now() - start}ms)`)
+  } catch (err) {
+    printRow(label, `${chalk.red(`failure: ${err.message}`)} (${Date.now() - start}ms)`)
+  }
+
+  print(` - ${url} (${ip})\n`)
+}
+
 export const wrapMethodCall = async (label: string, method: any) => {
   const start = Date.now()
   try {

packages/bp/src/typings/global.d.ts

@@ -30,6 +30,7 @@ declare namespace NodeJS {
     PORT: number
     STUDIO_PORT: number
     MESSAGING_PORT: number
+    NLU_ENDPOINT?: string
     NLU_PORT: number
     PROXY?: string
     EXTERNAL_URL: string