From 13ed91b707cc34b7a65eb3dfd37014d22f11c51a Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 4 Jun 2021 18:28:41 +0200 Subject: [PATCH] Build healthcheck into index.js (#455) * Build healthcheck into index.js * Add "Alternative modes" to README * Fix uncaught reject in Sentry flush --- Dockerfile | 2 +- README.md | 13 +++++++++++++ src/healthcheck.ts | 34 ++++++++++++++++++++++------------ src/index.ts | 9 ++++++++- src/main/pluginsServer.ts | 2 +- src/utils/utils.ts | 11 ++++++++++- 6 files changed, 55 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index aec48a38..9a437af9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,6 @@ FROM node:14 AS runner WORKDIR /code/ COPY --from=builder /code/ ./ -HEALTHCHECK --start-period=10s CMD [ "node", "dist/healthcheck.js" ] +HEALTHCHECK --start-period=10s CMD [ "node", "dist/index.js", "--healthcheck" ] CMD [ "node", "dist/index.js" ] diff --git a/README.md b/README.md index 33c2b5cc..d0175f1e 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,19 @@ Let's get you developing the plugin server in no time: 1. Run Postgres pipeline tests with `yarn test:postgres:{1,2}`. Run ClickHouse pipeline tests with `yarn test:clickhouse:{1,2}`. Run benchmarks with `yarn benchmark`. +## Alternative modes + +This program's main mode of operation is processing PostHog events, but there are also a few alternative utility ones. +Each one does a single thing. They are listed in the table below, in order of precedence. + +| Name | Description | CLI flags | +| ----------- | ---------------------------------------------------------- | ----------------- | +| Help | Show plugin server [configuration options](#configuration) | `-h`, `--help` | +| Version | Only show currently running plugin server version | `-v`, `--version` | +| Healthcheck | Check plugin server health and exit with 0 or 1 | `--healthcheck` | +| Migrate | Migrate Graphile job queue | `--migrate` | +| Idle | Start server in a completely idle, non-processing mode | `--idle` | + ## Configuration There's a multitude of settings you can use to control the plugin server. Use them as environment variables. diff --git a/src/healthcheck.ts b/src/healthcheck.ts index 541dd1bc..1352a763 100644 --- a/src/healthcheck.ts +++ b/src/healthcheck.ts @@ -1,16 +1,26 @@ import { defaultConfig } from './config/config' -import { Status } from './utils/status' +import { status } from './utils/status' import { createRedis } from './utils/utils' -const healthStatus = new Status('HLTH') - -void createRedis(defaultConfig).then(async (redis) => { - const ping = await redis.get('@posthog-plugin-server/ping') - if (ping) { - healthStatus.info('💚', `Redis key @posthog-plugin-server/ping found with value ${ping}`) - process.exit(0) - } else { - healthStatus.error('💔', 'Redis key @posthog-plugin-server/ping not found! Plugin server seems to be offline') - process.exit(1) +export async function healthcheck(): Promise { + const redis = await createRedis(defaultConfig) + try { + const ping = await redis.get('@posthog-plugin-server/ping') + if (ping) { + status.info('💚', `Redis key @posthog-plugin-server/ping found with value ${ping}`) + return true + } else { + status.error('💔', 'Redis key @posthog-plugin-server/ping not found! Plugin server seems to be offline') + return false + } + } catch (error) { + status.error('💥', 'An unexpected error occurred:', error) + return false + } finally { + redis.disconnect() } -}) +} + +export async function healthcheckWithExit(): Promise { + process.exit((await healthcheck()) ? 0 : 1) +} diff --git a/src/index.ts b/src/index.ts index a2e42a9b..44da4ee7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,5 @@ import { defaultConfig, formatConfigHelp } from './config/config' +import { healthcheckWithExit } from './healthcheck' import { initApp } from './init' import { GraphileQueue } from './main/job-queues/concurrent/graphile-queue' import { startPluginsServer } from './main/pluginsServer' @@ -11,8 +12,9 @@ const { argv } = process enum AlternativeMode { Help = 'HELP', Version = 'VRSN', + Healthcheck = 'HLTH', Idle = 'IDLE', - Migrate = 'MIGRATE', + Migrate = 'MGRT', } let alternativeMode: AlternativeMode | undefined @@ -20,6 +22,8 @@ if (argv.includes('--help') || argv.includes('-h')) { alternativeMode = AlternativeMode.Help } else if (argv.includes('--version') || argv.includes('-v')) { alternativeMode = AlternativeMode.Version +} else if (argv.includes('--healthcheck')) { + alternativeMode = AlternativeMode.Healthcheck } else if (argv.includes('--migrate')) { alternativeMode = AlternativeMode.Migrate } else if (defaultConfig.PLUGIN_SERVER_IDLE) { @@ -36,6 +40,9 @@ switch (alternativeMode) { case AlternativeMode.Help: status.info('⚙️', `Supported configuration environment variables:\n${formatConfigHelp(7)}`) break + case AlternativeMode.Healthcheck: + void healthcheckWithExit() + break case AlternativeMode.Idle: status.info('💤', `Disengaging this plugin server instance due to the PLUGIN_SERVER_IDLE env var...`) setInterval(() => { diff --git a/src/main/pluginsServer.ts b/src/main/pluginsServer.ts index 39110c80..9c6a4274 100644 --- a/src/main/pluginsServer.ts +++ b/src/main/pluginsServer.ts @@ -230,7 +230,7 @@ export async function startPluginsServer( } catch (error) { Sentry.captureException(error) status.error('💥', 'Launchpad failure!', error) - void Sentry.flush() // flush in the background + void Sentry.flush().catch(() => null) // Flush Sentry in the background await closeJobs() process.exit(1) } diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 4fe474ae..18fe0cba 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -15,6 +15,8 @@ import { status } from './status' /** Time until autoexit (due to error) gives up on graceful exit and kills the process right away. */ const GRACEFUL_EXIT_PERIOD_SECONDS = 5 +/** Number of Redis error events until the server is killed gracefully. */ +const REDIS_ERROR_COUNTER_LIMIT = 10 export function killGracefully(): void { status.error('⏲', 'Shutting plugin server down gracefully with SIGTERM...') @@ -386,10 +388,17 @@ export async function createRedis(serverConfig: PluginsServerConfig): Promise { + errorCounter++ Sentry.captureException(error) - status.error('🔴', 'Redis error encountered! Trying to reconnect...\n', error) + if (errorCounter > REDIS_ERROR_COUNTER_LIMIT) { + status.error('😡', 'Redis error encountered! Enough of this, I quit!\n', error) + killGracefully() + } else { + status.error('🔴', 'Redis error encountered! Trying to reconnect...\n', error) + } }) .on('ready', () => { if (process.env.NODE_ENV !== 'test') {