Build healthcheck into index.js (#455)

* Build healthcheck into index.js

* Add "Alternative modes" to README

* Fix uncaught reject in Sentry flush

Dockerfile

@@ -14,6 +14,6 @@ FROM node:14 AS runner
 WORKDIR /code/
 COPY --from=builder /code/ ./
 
-HEALTHCHECK --start-period=10s CMD [ "node", "dist/healthcheck.js" ]
+HEALTHCHECK --start-period=10s CMD [ "node", "dist/index.js", "--healthcheck" ]
 
 CMD [ "node", "dist/index.js" ]

README.md

@@ -23,6 +23,19 @@ Let's get you developing the plugin server in no time:
 
 1. Run Postgres pipeline tests with `yarn test:postgres:{1,2}`. Run ClickHouse pipeline tests with `yarn test:clickhouse:{1,2}`. Run benchmarks with `yarn benchmark`.
 
+## Alternative modes
+
+This program's main mode of operation is processing PostHog events, but there are also a few alternative utility ones.
+Each one does a single thing. They are listed in the table below, in order of precedence.
+
+| Name        | Description                                                | CLI flags         |
+| ----------- | ---------------------------------------------------------- | ----------------- |
+| Help        | Show plugin server [configuration options](#configuration) | `-h`, `--help`    |
+| Version     | Only show currently running plugin server version          | `-v`, `--version` |
+| Healthcheck | Check plugin server health and exit with 0 or 1            | `--healthcheck`   |
+| Migrate     | Migrate Graphile job queue                                 | `--migrate`       |
+| Idle        | Start server in a completely idle, non-processing mode     | `--idle`          |
+
 ## Configuration
 
 There's a multitude of settings you can use to control the plugin server. Use them as environment variables.

src/healthcheck.ts

@@ -1,16 +1,26 @@
 import { defaultConfig } from './config/config'
-import { Status } from './utils/status'
+import { status } from './utils/status'
 import { createRedis } from './utils/utils'
 
-const healthStatus = new Status('HLTH')
-
-void createRedis(defaultConfig).then(async (redis) => {
-    const ping = await redis.get('@posthog-plugin-server/ping')
-    if (ping) {
-        healthStatus.info('💚', `Redis key @posthog-plugin-server/ping found with value ${ping}`)
-        process.exit(0)
-    } else {
-        healthStatus.error('💔', 'Redis key @posthog-plugin-server/ping not found! Plugin server seems to be offline')
-        process.exit(1)
+export async function healthcheck(): Promise<boolean> {
+    const redis = await createRedis(defaultConfig)
+    try {
+        const ping = await redis.get('@posthog-plugin-server/ping')
+        if (ping) {
+            status.info('💚', `Redis key @posthog-plugin-server/ping found with value ${ping}`)
+            return true
+        } else {
+            status.error('💔', 'Redis key @posthog-plugin-server/ping not found! Plugin server seems to be offline')
+            return false
+        }
+    } catch (error) {
+        status.error('💥', 'An unexpected error occurred:', error)
+        return false
+    } finally {
+        redis.disconnect()
     }
-})
+}
+
+export async function healthcheckWithExit(): Promise<never> {
+    process.exit((await healthcheck()) ? 0 : 1)
+}

src/index.ts

@@ -1,4 +1,5 @@
 import { defaultConfig, formatConfigHelp } from './config/config'
+import { healthcheckWithExit } from './healthcheck'
 import { initApp } from './init'
 import { GraphileQueue } from './main/job-queues/concurrent/graphile-queue'
 import { startPluginsServer } from './main/pluginsServer'
@@ -11,15 +12,18 @@ const { argv } = process
 enum AlternativeMode {
     Help = 'HELP',
     Version = 'VRSN',
+    Healthcheck = 'HLTH',
     Idle = 'IDLE',
-    Migrate = 'MIGRATE',
+    Migrate = 'MGRT',
 }
 
 let alternativeMode: AlternativeMode | undefined
 if (argv.includes('--help') || argv.includes('-h')) {
     alternativeMode = AlternativeMode.Help
 } else if (argv.includes('--version') || argv.includes('-v')) {
     alternativeMode = AlternativeMode.Version
+} else if (argv.includes('--healthcheck')) {
+    alternativeMode = AlternativeMode.Healthcheck
 } else if (argv.includes('--migrate')) {
     alternativeMode = AlternativeMode.Migrate
 } else if (defaultConfig.PLUGIN_SERVER_IDLE) {
@@ -36,6 +40,9 @@ switch (alternativeMode) {
     case AlternativeMode.Help:
         status.info('⚙️', `Supported configuration environment variables:\n${formatConfigHelp(7)}`)
         break
+    case AlternativeMode.Healthcheck:
+        void healthcheckWithExit()
+        break
     case AlternativeMode.Idle:
         status.info('💤', `Disengaging this plugin server instance due to the PLUGIN_SERVER_IDLE env var...`)
         setInterval(() => {

src/main/pluginsServer.ts

@@ -230,7 +230,7 @@ export async function startPluginsServer(
     } catch (error) {
         Sentry.captureException(error)
         status.error('💥', 'Launchpad failure!', error)
-        void Sentry.flush() // flush in the background
+        void Sentry.flush().catch(() => null) // Flush Sentry in the background
         await closeJobs()
         process.exit(1)
     }

src/utils/utils.ts

@@ -15,6 +15,8 @@ import { status } from './status'
 
 /** Time until autoexit (due to error) gives up on graceful exit and kills the process right away. */
 const GRACEFUL_EXIT_PERIOD_SECONDS = 5
+/** Number of Redis error events until the server is killed gracefully. */
+const REDIS_ERROR_COUNTER_LIMIT = 10
 
 export function killGracefully(): void {
     status.error('⏲', 'Shutting plugin server down gracefully with SIGTERM...')
@@ -386,10 +388,17 @@ export async function createRedis(serverConfig: PluginsServerConfig): Promise<Re
         ...credentials,
         maxRetriesPerRequest: -1,
     })
+    let errorCounter = 0
     redis
         .on('error', (error) => {
+            errorCounter++
             Sentry.captureException(error)
-            status.error('🔴', 'Redis error encountered! Trying to reconnect...\n', error)
+            if (errorCounter > REDIS_ERROR_COUNTER_LIMIT) {
+                status.error('😡', 'Redis error encountered! Enough of this, I quit!\n', error)
+                killGracefully()
+            } else {
+                status.error('🔴', 'Redis error encountered! Trying to reconnect...\n', error)
+            }
         })
         .on('ready', () => {
             if (process.env.NODE_ENV !== 'test') {