From 419cd20b4e2e5c5399cad52f820340212ed0da91 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 5 Jun 2023 14:05:34 -0400 Subject: [PATCH] [8.8] Fix config stacking order (#158827) (#159025) # Backport This will backport the following commits from `main` to `8.8`: - [Fix config stacking order (#158827)](https://github.com/elastic/kibana/pull/158827) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) --------- Co-authored-by: Alex Szabo --- config/README.md | 9 +- .../src/bootstrap.test.mocks.ts | 1 + .../src/bootstrap.ts | 4 +- .../integration_tests/config_ordering.test.ts | 243 ++++++++++++++++++ src/cli/serve/serve.js | 9 +- 5 files changed, 261 insertions(+), 5 deletions(-) create mode 100644 src/cli/serve/integration_tests/config_ordering.test.ts diff --git a/config/README.md b/config/README.md index b5cad71cb0813e..83ef1b1c661205 100644 --- a/config/README.md +++ b/config/README.md @@ -5,9 +5,12 @@ this configuration, pass `--serverless={mode}` or run `yarn serverless-{mode}` valid modes are currently: `es`, `oblt`, and `security` configuration is applied in the following order, later values override - 1. kibana.yml - 2. serverless.yml - 3. serverless.{mode}.yml + 1. serverless.yml (serverless configs go first) + 2. serverless.{mode}.yml (serverless configs go first) + 3. base config, in this preference order: + - my-config.yml(s) (set by --config) + - env-config.yml (described by `env.KBN_CONFIG_PATHS`) + - kibana.yml (default @ `env.KBN_PATH_CONF`/kibana.yml) 4. kibana.dev.yml 5. serverless.dev.yml 6. serverless.{mode}.dev.yml diff --git a/packages/core/root/core-root-server-internal/src/bootstrap.test.mocks.ts b/packages/core/root/core-root-server-internal/src/bootstrap.test.mocks.ts index 07277d565f694c..be54edf7f2124e 100644 --- a/packages/core/root/core-root-server-internal/src/bootstrap.test.mocks.ts +++ b/packages/core/root/core-root-server-internal/src/bootstrap.test.mocks.ts @@ -21,5 +21,6 @@ jest.doMock('@kbn/config', () => ({ jest.doMock('./root', () => ({ Root: jest.fn(() => ({ shutdown: jest.fn(), + logger: { get: () => ({ info: jest.fn(), debug: jest.fn() }) }, })), })); diff --git a/packages/core/root/core-root-server-internal/src/bootstrap.ts b/packages/core/root/core-root-server-internal/src/bootstrap.ts index 3f55b6493a6bda..bb0e3ddc8c7011 100644 --- a/packages/core/root/core-root-server-internal/src/bootstrap.ts +++ b/packages/core/root/core-root-server-internal/src/bootstrap.ts @@ -78,6 +78,9 @@ export async function bootstrap({ configs, cliArgs, applyConfigOverrides }: Boot } const root = new Root(rawConfigService, env, onRootShutdown); + const cliLogger = root.logger.get('cli'); + + cliLogger.debug('Kibana configurations evaluated in this order: ' + env.configs.join(', ')); process.on('SIGHUP', () => reloadConfiguration()); @@ -93,7 +96,6 @@ export async function bootstrap({ configs, cliArgs, applyConfigOverrides }: Boot }); function reloadConfiguration(reason = 'SIGHUP signal received') { - const cliLogger = root.logger.get('cli'); cliLogger.info(`Reloading Kibana configuration (reason: ${reason}).`, { tags: ['config'] }); try { diff --git a/src/cli/serve/integration_tests/config_ordering.test.ts b/src/cli/serve/integration_tests/config_ordering.test.ts new file mode 100644 index 00000000000000..692c402cab8702 --- /dev/null +++ b/src/cli/serve/integration_tests/config_ordering.test.ts @@ -0,0 +1,243 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Fs from 'fs'; +import * as Path from 'path'; +import * as Os from 'os'; +import * as Child from 'child_process'; +import Del from 'del'; +import * as Rx from 'rxjs'; +import { filter, map, take, timeout } from 'rxjs/operators'; + +const tempDir = Path.join(Os.tmpdir(), 'kbn-config-test'); + +const kibanaPath = follow('../../../../scripts/kibana.js'); + +const TIMEOUT_MS = 20000; + +const envForTempDir = { + env: { KBN_PATH_CONF: tempDir }, +}; + +const TestFiles = { + fileList: [] as string[], + + createEmptyConfigFiles(fileNames: string[], root: string = tempDir): string[] { + const configFiles = []; + for (const fileName of fileNames) { + const filePath = Path.resolve(root, fileName); + + if (!Fs.existsSync(filePath)) { + Fs.writeFileSync(filePath, 'dummy'); + + TestFiles.fileList.push(filePath); + } + + configFiles.push(filePath); + } + + return configFiles; + }, + cleanUpEmptyConfigFiles() { + for (const filePath of TestFiles.fileList) { + Del.sync(filePath); + } + TestFiles.fileList.length = 0; + }, +}; + +describe('Server configuration ordering', () => { + let kibanaProcess: Child.ChildProcessWithoutNullStreams; + + beforeEach(() => { + Fs.mkdirSync(tempDir, { recursive: true }); + }); + + afterEach(async () => { + if (kibanaProcess !== undefined) { + const exitPromise = new Promise((resolve) => kibanaProcess?.once('exit', resolve)); + kibanaProcess.kill('SIGKILL'); + await exitPromise; + } + + Del.sync(tempDir, { force: true }); + TestFiles.cleanUpEmptyConfigFiles(); + }); + + it('loads default config set without any options', async function () { + TestFiles.createEmptyConfigFiles(['kibana.yml']); + + kibanaProcess = Child.spawn(process.execPath, [kibanaPath, '--verbose'], envForTempDir); + const configList = await extractConfigurationOrder(kibanaProcess); + + expect(configList).toEqual(['kibana.yml']); + }); + + it('loads serverless configs when --serverless is set', async () => { + TestFiles.createEmptyConfigFiles([ + 'serverless.yml', + 'serverless.oblt.yml', + 'kibana.yml', + 'serverless.recent.yml', + ]); + + kibanaProcess = Child.spawn( + process.execPath, + [kibanaPath, '--verbose', '--serverless', 'oblt'], + envForTempDir + ); + const configList = await extractConfigurationOrder(kibanaProcess); + + expect(configList).toEqual([ + 'serverless.yml', + 'serverless.oblt.yml', + 'kibana.yml', + 'serverless.recent.yml', + ]); + }); + + it('prefers --config options over default', async () => { + const [configPath] = TestFiles.createEmptyConfigFiles([ + 'potato.yml', + 'serverless.yml', + 'serverless.oblt.yml', + 'kibana.yml', + 'serverless.recent.yml', + ]); + + kibanaProcess = Child.spawn( + process.execPath, + [kibanaPath, '--verbose', '--serverless', 'oblt', '--config', configPath], + envForTempDir + ); + const configList = await extractConfigurationOrder(kibanaProcess); + + expect(configList).toEqual([ + 'serverless.yml', + 'serverless.oblt.yml', + 'potato.yml', + 'serverless.recent.yml', + ]); + }); + + it('defaults to "es" if --serverless and --dev are there', async () => { + TestFiles.createEmptyConfigFiles([ + 'serverless.yml', + 'serverless.es.yml', + 'kibana.yml', + 'kibana.dev.yml', + 'serverless.dev.yml', + ]); + + kibanaProcess = Child.spawn( + process.execPath, + [kibanaPath, '--verbose', '--serverless', '--dev'], + envForTempDir + ); + const configList = await extractConfigurationOrder(kibanaProcess); + + expect(configList).toEqual([ + 'serverless.yml', + 'serverless.es.yml', + 'kibana.yml', + 'serverless.recent.yml', + 'kibana.dev.yml', + 'serverless.dev.yml', + ]); + }); + + it('adds dev configs to the stack', async () => { + TestFiles.createEmptyConfigFiles([ + 'serverless.yml', + 'serverless.security.yml', + 'serverless.recent.yml', + 'kibana.yml', + 'kibana.dev.yml', + 'serverless.dev.yml', + ]); + + kibanaProcess = Child.spawn( + process.execPath, + [kibanaPath, '--verbose', '--serverless', 'security', '--dev'], + envForTempDir + ); + + const configList = await extractConfigurationOrder(kibanaProcess); + + expect(configList).toEqual([ + 'serverless.yml', + 'serverless.security.yml', + 'kibana.yml', + 'serverless.recent.yml', + 'kibana.dev.yml', + 'serverless.dev.yml', + ]); + }); +}); + +async function extractConfigurationOrder( + proc: Child.ChildProcessWithoutNullStreams +): Promise { + const configMessage = await waitForMessage(proc, /[Cc]onfig.*order:/, TIMEOUT_MS); + + const configList = configMessage + .match(/order: (.*)$/) + ?.at(1) + ?.split(', ') + ?.map((path) => Path.basename(path)); + + return configList; +} + +async function waitForMessage( + proc: Child.ChildProcessWithoutNullStreams, + expression: string | RegExp, + timeoutMs: number +): Promise { + const message$ = Rx.fromEvent(proc.stdout!, 'data').pipe( + map((messages) => String(messages).split('\n').filter(Boolean)) + ); + + const trackedExpression$ = message$.pipe( + // We know the sighup handler will be registered before this message logged + filter((messages: string[]) => messages.some((m) => m.match(expression))), + take(1) + ); + + const error$ = message$.pipe( + filter((messages: string[]) => messages.some((line) => line.match(/fatal/i))), + take(1), + map((line) => new Error(line.join('\n'))) + ); + + const value = await Rx.firstValueFrom( + Rx.race(trackedExpression$, error$).pipe( + timeout({ + first: timeoutMs, + with: () => + Rx.throwError( + () => new Error(`Config options didn't appear in logs for ${timeoutMs / 1000}s...`) + ), + }) + ) + ); + + if (value instanceof Error) { + throw value; + } + + if (Array.isArray(value)) { + return value[0]; + } else { + return value; + } +} + +function follow(file: string) { + return Path.relative(process.cwd(), Path.resolve(__dirname, file)); +} diff --git a/src/cli/serve/serve.js b/src/cli/serve/serve.js index c1b9f04fd8d814..581b6562227e45 100644 --- a/src/cli/serve/serve.js +++ b/src/cli/serve/serve.js @@ -20,6 +20,8 @@ import { readKeystore } from '../keystore/read_keystore'; /** @type {ServerlessProjectMode[]} */ const VALID_SERVERLESS_PROJECT_MODE = ['es', 'oblt', 'security']; +const isNotEmpty = _.negate(_.isEmpty); + /** * @param {Record} opts * @returns {ServerlessProjectMode | true | null} @@ -311,8 +313,13 @@ export default function (program) { } command.action(async function (opts) { + const cliConfigs = opts.config || []; + const envConfigs = getEnvConfigs(); + const defaultConfig = getConfigPath(); + + const configs = [cliConfigs, envConfigs, [defaultConfig]].find(isNotEmpty); + const unknownOptions = this.getUnknownOptions(); - const configs = [getConfigPath(), ...getEnvConfigs(), ...(opts.config || [])]; const serverlessMode = getServerlessProjectMode(opts); if (serverlessMode) {