From 1c0e76bb3f563356961b09b384d7ba055f6bafd0 Mon Sep 17 00:00:00 2001 From: John Schulz Date: Tue, 5 May 2020 20:56:12 -0400 Subject: [PATCH] [Ingest Manager] Use cloudId for ES & Kibana URLs if available. (#65366) (#65393) * Use cloudId for ES & Kibana URLs if available. * Add tests and a missing return * Use console.debug for consistency --- .../common/services/decode_cloud_id.test.ts | 100 ++++++++++++++++++ .../common/services/decode_cloud_id.ts | 65 ++++++++++++ .../ingest_manager/common/services/index.ts | 1 + .../ingest_manager/common/types/index.ts | 1 - x-pack/plugins/ingest_manager/server/index.ts | 2 +- .../ingest_manager/server/services/output.ts | 9 +- .../ingest_manager/server/services/setup.ts | 9 +- 7 files changed, 182 insertions(+), 5 deletions(-) create mode 100644 x-pack/plugins/ingest_manager/common/services/decode_cloud_id.test.ts create mode 100644 x-pack/plugins/ingest_manager/common/services/decode_cloud_id.ts diff --git a/x-pack/plugins/ingest_manager/common/services/decode_cloud_id.test.ts b/x-pack/plugins/ingest_manager/common/services/decode_cloud_id.test.ts new file mode 100644 index 00000000000000..dcec54f47440ae --- /dev/null +++ b/x-pack/plugins/ingest_manager/common/services/decode_cloud_id.test.ts @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { decodeCloudId } from './decode_cloud_id'; + +describe('Ingest Manager - decodeCloudId', () => { + it('parses various CloudID formats', () => { + const tests = [ + { + cloudID: + 'staging:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRjZWM2ZjI2MWE3NGJmMjRjZTMzYmI4ODExYjg0Mjk0ZiRjNmMyY2E2ZDA0MjI0OWFmMGNjN2Q3YTllOTYyNTc0Mw==', + expectedEsURL: 'https://cec6f261a74bf24ce33bb8811b84294f.us-east-1.aws.found.io:443', + expectedKibanaURL: 'https://c6c2ca6d042249af0cc7d7a9e9625743.us-east-1.aws.found.io:443', + }, + { + cloudID: + 'dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRjZWM2ZjI2MWE3NGJmMjRjZTMzYmI4ODExYjg0Mjk0ZiRjNmMyY2E2ZDA0MjI0OWFmMGNjN2Q3YTllOTYyNTc0Mw==', + expectedEsURL: 'https://cec6f261a74bf24ce33bb8811b84294f.us-east-1.aws.found.io:443', + expectedKibanaURL: 'https://c6c2ca6d042249af0cc7d7a9e9625743.us-east-1.aws.found.io:443', + }, + { + cloudID: + ':dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRjZWM2ZjI2MWE3NGJmMjRjZTMzYmI4ODExYjg0Mjk0ZiRjNmMyY2E2ZDA0MjI0OWFmMGNjN2Q3YTllOTYyNTc0Mw==', + expectedEsURL: 'https://cec6f261a74bf24ce33bb8811b84294f.us-east-1.aws.found.io:443', + expectedKibanaURL: 'https://c6c2ca6d042249af0cc7d7a9e9625743.us-east-1.aws.found.io:443', + }, + { + cloudID: + 'gcp-cluster:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJDhhMDI4M2FmMDQxZjE5NWY3NzI5YmMwNGM2NmEwZmNlJDBjZDVjZDU2OGVlYmU1M2M4OWViN2NhZTViYWM4YjM3', + expectedEsURL: 'https://8a0283af041f195f7729bc04c66a0fce.us-central1.gcp.cloud.es.io:443', + expectedKibanaURL: + 'https://0cd5cd568eebe53c89eb7cae5bac8b37.us-central1.gcp.cloud.es.io:443', + }, + { + cloudID: + 'custom-port:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvOjkyNDMkYWMzMWViYjkwMjQxNzczMTU3MDQzYzM0ZmQyNmZkNDYkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA=', + expectedEsURL: 'https://ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io:9243', + expectedKibanaURL: + 'https://a4c06230e48c8fce7be88a074a3bb3e0.us-central1.gcp.cloud.es.io:9243', + }, + { + cloudID: + 'different-es-kb-port:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA==', + expectedEsURL: 'https://ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io:9243', + expectedKibanaURL: + 'https://a4c06230e48c8fce7be88a074a3bb3e0.us-central1.gcp.cloud.es.io:9244', + }, + { + cloudID: + 'only-kb-set:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2JGE0YzA2MjMwZTQ4YzhmY2U3YmU4OGEwNzRhM2JiM2UwOjkyNDQ=', + expectedEsURL: 'https://ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io:443', + expectedKibanaURL: + 'https://a4c06230e48c8fce7be88a074a3bb3e0.us-central1.gcp.cloud.es.io:9244', + }, + { + cloudID: + 'host-and-kb-set:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvOjkyNDMkYWMzMWViYjkwMjQxNzczMTU3MDQzYzM0ZmQyNmZkNDYkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA==', + expectedEsURL: 'https://ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io:9243', + expectedKibanaURL: + 'https://a4c06230e48c8fce7be88a074a3bb3e0.us-central1.gcp.cloud.es.io:9244', + }, + { + cloudID: + 'extra-items:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2JGE0YzA2MjMwZTQ4YzhmY2U3YmU4OGEwNzRhM2JiM2UwJGFub3RoZXJpZCRhbmRhbm90aGVy', + expectedEsURL: 'https://ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io:443', + expectedKibanaURL: + 'https://a4c06230e48c8fce7be88a074a3bb3e0.us-central1.gcp.cloud.es.io:443', + }, + ]; + + for (const test of tests) { + const decoded = decodeCloudId(test.cloudID); + expect(decoded).toBeTruthy(); + expect(decoded?.elasticsearchUrl === test.expectedEsURL).toBe(true); + expect(decoded?.kibanaUrl === test.expectedKibanaURL).toBe(true); + } + }); + + it('returns undefined for invalid formats', () => { + const tests = [ + { + cloudID: + 'staging:garbagedXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRjZWM2ZjI2MWE3NGJmMjRjZTMzYmI4ODExYjg0Mjk0ZiRjNmMyY2E2ZDA0MjI0OWFmMGNjN2Q3YTllOTYyNTc0Mw==', + errorMsg: 'base64 decoding failed', + }, + { + cloudID: 'dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJDhhMDI4M2FmMDQxZjE5NWY3NzI5YmMwNGM2NmEwZg==', + errorMsg: 'Expected at least 3 parts', + }, + ]; + + for (const test of tests) { + const decoded = decodeCloudId(test.cloudID); + expect(decoded).toBe(undefined); + // decodeCloudId currently only logs; not throws errors + } + }); +}); diff --git a/x-pack/plugins/ingest_manager/common/services/decode_cloud_id.ts b/x-pack/plugins/ingest_manager/common/services/decode_cloud_id.ts new file mode 100644 index 00000000000000..67389e554ed243 --- /dev/null +++ b/x-pack/plugins/ingest_manager/common/services/decode_cloud_id.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// decodeCloudId decodes the c.id into c.esURL and c.kibURL +export function decodeCloudId( + cid: string +): + | { + host: string; + defaultPort: string; + elasticsearchUrl: string; + kibanaUrl: string; + } + | undefined { + // 1. Ignore anything before `:`. + const id = cid.split(':').pop(); + if (!id) { + // throw new Error(`Unable to decode ${id}`); + // eslint-disable-next-line no-console + console.debug(`Unable to decode ${id}`); + return; + } + + // 2. base64 decode + let decoded: string | undefined; + try { + decoded = Buffer.from(id, 'base64').toString('utf8'); + } catch { + // throw new Error(`base64 decoding failed on ${id}`); + // eslint-disable-next-line no-console + console.debug(`base64 decoding failed on ${id}`); + return; + } + + // 3. separate based on `$` + const words = decoded.split('$'); + if (words.length < 3) { + // throw new Error(`Expected at least 3 parts in ${decoded}`); + // eslint-disable-next-line no-console + console.debug(`Expected at least 3 parts in ${decoded}`); + return; + } + // 4. extract port from the ES and Kibana host + const [host, defaultPort] = extractPortFromName(words[0]); + const [esId, esPort] = extractPortFromName(words[1], defaultPort); + const [kbId, kbPort] = extractPortFromName(words[2], defaultPort); + // 5. form the URLs + const esUrl = `https://${esId}.${host}:${esPort}`; + const kbUrl = `https://${kbId}.${host}:${kbPort}`; + return { + host, + defaultPort, + elasticsearchUrl: esUrl, + kibanaUrl: kbUrl, + }; +} +// extractPortFromName takes a string in the form `id:port` and returns the +// Id and the port. If there's no `:`, the default port is returned +function extractPortFromName(word: string, defaultPort = '443') { + const [host, port = defaultPort] = word.split(':'); + return [host, port]; +} diff --git a/x-pack/plugins/ingest_manager/common/services/index.ts b/x-pack/plugins/ingest_manager/common/services/index.ts index 8e704bb717257d..91dbbdd515c3e8 100644 --- a/x-pack/plugins/ingest_manager/common/services/index.ts +++ b/x-pack/plugins/ingest_manager/common/services/index.ts @@ -9,3 +9,4 @@ export * from './routes'; export { packageToConfigDatasourceInputs, packageToConfigDatasource } from './package_to_config'; export { storedDatasourceToAgentDatasource } from './datasource_to_agent_datasource'; export { AgentStatusKueryHelper }; +export { decodeCloudId } from './decode_cloud_id'; diff --git a/x-pack/plugins/ingest_manager/common/types/index.ts b/x-pack/plugins/ingest_manager/common/types/index.ts index b357d0c2d75f48..800e77a6a27427 100644 --- a/x-pack/plugins/ingest_manager/common/types/index.ts +++ b/x-pack/plugins/ingest_manager/common/types/index.ts @@ -15,7 +15,6 @@ export interface IngestManagerConfigType { fleet: { enabled: boolean; tlsCheckDisabled: boolean; - defaultOutputHost: string; kibana: { host?: string; ca_sha256?: string; diff --git a/x-pack/plugins/ingest_manager/server/index.ts b/x-pack/plugins/ingest_manager/server/index.ts index 6096af8d808010..84408b60d3edb1 100644 --- a/x-pack/plugins/ingest_manager/server/index.ts +++ b/x-pack/plugins/ingest_manager/server/index.ts @@ -32,7 +32,7 @@ export const config = { ca_sha256: schema.maybe(schema.string()), }), elasticsearch: schema.object({ - host: schema.string({ defaultValue: 'http://localhost:9200' }), + host: schema.maybe(schema.string()), ca_sha256: schema.maybe(schema.string()), }), }), diff --git a/x-pack/plugins/ingest_manager/server/services/output.ts b/x-pack/plugins/ingest_manager/server/services/output.ts index 3628c5bd9e1830..ce6f1f2e27130b 100644 --- a/x-pack/plugins/ingest_manager/server/services/output.ts +++ b/x-pack/plugins/ingest_manager/server/services/output.ts @@ -7,6 +7,7 @@ import { SavedObjectsClientContract } from 'src/core/server'; import { NewOutput, Output } from '../types'; import { DEFAULT_OUTPUT, OUTPUT_SAVED_OBJECT_TYPE } from '../constants'; import { appContextService } from './app_context'; +import { decodeCloudId } from '../../common'; const SAVED_OBJECT_TYPE = OUTPUT_SAVED_OBJECT_TYPE; @@ -16,11 +17,17 @@ class OutputService { type: OUTPUT_SAVED_OBJECT_TYPE, filter: `${OUTPUT_SAVED_OBJECT_TYPE}.attributes.is_default:true`, }); + const cloud = appContextService.getCloud(); + const cloudId = cloud?.isCloudEnabled && cloud.cloudId; + const cloudUrl = cloudId && decodeCloudId(cloudId)?.elasticsearchUrl; + const flagsUrl = appContextService.getConfig()!.fleet.elasticsearch.host; + const defaultUrl = 'http://localhost:9200'; + const defaultOutputUrl = cloudUrl || flagsUrl || defaultUrl; if (!outputs.saved_objects.length) { const newDefaultOutput = { ...DEFAULT_OUTPUT, - hosts: [appContextService.getConfig()!.fleet.elasticsearch.host], + hosts: [defaultOutputUrl], ca_sha256: appContextService.getConfig()!.fleet.elasticsearch.ca_sha256, } as NewOutput; diff --git a/x-pack/plugins/ingest_manager/server/services/setup.ts b/x-pack/plugins/ingest_manager/server/services/setup.ts index 22acce8d4a51c7..14837d9edd1ccb 100644 --- a/x-pack/plugins/ingest_manager/server/services/setup.ts +++ b/x-pack/plugins/ingest_manager/server/services/setup.ts @@ -18,6 +18,7 @@ import { Installation, Output, DEFAULT_AGENT_CONFIGS_PACKAGES, + decodeCloudId, } from '../../common'; import { getPackageInfo } from './epm/packages'; import { datasourceService } from './datasource'; @@ -43,7 +44,11 @@ export async function setupIngestManager( const serverInfo = http.getServerInfo(); const basePath = http.basePath; - const defaultKibanaUrl = url.format({ + const cloud = appContextService.getCloud(); + const cloudId = cloud?.isCloudEnabled && cloud.cloudId; + const cloudUrl = cloudId && decodeCloudId(cloudId)?.kibanaUrl; + const flagsUrl = appContextService.getConfig()?.fleet?.kibana?.host; + const defaultUrl = url.format({ protocol: serverInfo.protocol, hostname: serverInfo.host, port: serverInfo.port, @@ -53,7 +58,7 @@ export async function setupIngestManager( return settingsService.saveSettings(soClient, { agent_auto_upgrade: true, package_auto_upgrade: true, - kibana_url: appContextService.getConfig()?.fleet?.kibana?.host ?? defaultKibanaUrl, + kibana_url: cloudUrl || flagsUrl || defaultUrl, }); }