[Ingest Manager] Use cloudId for ES & Kibana URLs if available. (#65366…

…) (#65393) * Use cloudId for ES & Kibana URLs if available. * Add tests and a missing return * Use console.debug for consistency
elastic · May 6, 2020 · 1c0e76b · 1c0e76b
1 parent 94ad6d6
commit 1c0e76b
Show file tree

Hide file tree

Showing 7 changed files with 182 additions and 5 deletions.
diff --git a/x-pack/plugins/ingest_manager/common/services/decode_cloud_id.test.ts b/x-pack/plugins/ingest_manager/common/services/decode_cloud_id.test.ts
@@ -0,0 +1,100 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { decodeCloudId } from './decode_cloud_id';
+
+describe('Ingest Manager - decodeCloudId', () => {
+  it('parses various CloudID formats', () => {
+    const tests = [
+      {
+        cloudID:
+          'staging:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRjZWM2ZjI2MWE3NGJmMjRjZTMzYmI4ODExYjg0Mjk0ZiRjNmMyY2E2ZDA0MjI0OWFmMGNjN2Q3YTllOTYyNTc0Mw==',
+        expectedEsURL: 'https://cec6f261a74bf24ce33bb8811b84294f.us-east-1.aws.found.io:443',
+        expectedKibanaURL: 'https://c6c2ca6d042249af0cc7d7a9e9625743.us-east-1.aws.found.io:443',
+      },
+      {
+        cloudID:
+          'dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRjZWM2ZjI2MWE3NGJmMjRjZTMzYmI4ODExYjg0Mjk0ZiRjNmMyY2E2ZDA0MjI0OWFmMGNjN2Q3YTllOTYyNTc0Mw==',
+        expectedEsURL: 'https://cec6f261a74bf24ce33bb8811b84294f.us-east-1.aws.found.io:443',
+        expectedKibanaURL: 'https://c6c2ca6d042249af0cc7d7a9e9625743.us-east-1.aws.found.io:443',
+      },
+      {
+        cloudID:
+          ':dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRjZWM2ZjI2MWE3NGJmMjRjZTMzYmI4ODExYjg0Mjk0ZiRjNmMyY2E2ZDA0MjI0OWFmMGNjN2Q3YTllOTYyNTc0Mw==',
+        expectedEsURL: 'https://cec6f261a74bf24ce33bb8811b84294f.us-east-1.aws.found.io:443',
+        expectedKibanaURL: 'https://c6c2ca6d042249af0cc7d7a9e9625743.us-east-1.aws.found.io:443',
+      },
+      {
+        cloudID:
+          'gcp-cluster:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJDhhMDI4M2FmMDQxZjE5NWY3NzI5YmMwNGM2NmEwZmNlJDBjZDVjZDU2OGVlYmU1M2M4OWViN2NhZTViYWM4YjM3',
+        expectedEsURL: 'https://8a0283af041f195f7729bc04c66a0fce.us-central1.gcp.cloud.es.io:443',
+        expectedKibanaURL:
+          'https://0cd5cd568eebe53c89eb7cae5bac8b37.us-central1.gcp.cloud.es.io:443',
+      },
+      {
+        cloudID:
+          'custom-port:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvOjkyNDMkYWMzMWViYjkwMjQxNzczMTU3MDQzYzM0ZmQyNmZkNDYkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA=',
+        expectedEsURL: 'https://ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io:9243',
+        expectedKibanaURL:
+          'https://a4c06230e48c8fce7be88a074a3bb3e0.us-central1.gcp.cloud.es.io:9243',
+      },
+      {
+        cloudID:
+          'different-es-kb-port:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA==',
+        expectedEsURL: 'https://ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io:9243',
+        expectedKibanaURL:
+          'https://a4c06230e48c8fce7be88a074a3bb3e0.us-central1.gcp.cloud.es.io:9244',
+      },
+      {
+        cloudID:
+          'only-kb-set:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2JGE0YzA2MjMwZTQ4YzhmY2U3YmU4OGEwNzRhM2JiM2UwOjkyNDQ=',
+        expectedEsURL: 'https://ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io:443',
+        expectedKibanaURL:
+          'https://a4c06230e48c8fce7be88a074a3bb3e0.us-central1.gcp.cloud.es.io:9244',
+      },
+      {
+        cloudID:
+          'host-and-kb-set:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvOjkyNDMkYWMzMWViYjkwMjQxNzczMTU3MDQzYzM0ZmQyNmZkNDYkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA==',
+        expectedEsURL: 'https://ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io:9243',
+        expectedKibanaURL:
+          'https://a4c06230e48c8fce7be88a074a3bb3e0.us-central1.gcp.cloud.es.io:9244',
+      },
+      {
+        cloudID:
+          'extra-items:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2JGE0YzA2MjMwZTQ4YzhmY2U3YmU4OGEwNzRhM2JiM2UwJGFub3RoZXJpZCRhbmRhbm90aGVy',
+        expectedEsURL: 'https://ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io:443',
+        expectedKibanaURL:
+          'https://a4c06230e48c8fce7be88a074a3bb3e0.us-central1.gcp.cloud.es.io:443',
+      },
+    ];
+
+    for (const test of tests) {
+      const decoded = decodeCloudId(test.cloudID);
+      expect(decoded).toBeTruthy();
+      expect(decoded?.elasticsearchUrl === test.expectedEsURL).toBe(true);
+      expect(decoded?.kibanaUrl === test.expectedKibanaURL).toBe(true);
+    }
+  });
+
+  it('returns undefined for invalid formats', () => {
+    const tests = [
+      {
+        cloudID:
+          'staging:garbagedXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRjZWM2ZjI2MWE3NGJmMjRjZTMzYmI4ODExYjg0Mjk0ZiRjNmMyY2E2ZDA0MjI0OWFmMGNjN2Q3YTllOTYyNTc0Mw==',
+        errorMsg: 'base64 decoding failed',
+      },
+      {
+        cloudID: 'dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJDhhMDI4M2FmMDQxZjE5NWY3NzI5YmMwNGM2NmEwZg==',
+        errorMsg: 'Expected at least 3 parts',
+      },
+    ];
+
+    for (const test of tests) {
+      const decoded = decodeCloudId(test.cloudID);
+      expect(decoded).toBe(undefined);
+      // decodeCloudId currently only logs; not throws errors
+    }
+  });
+});
diff --git a/x-pack/plugins/ingest_manager/common/services/decode_cloud_id.ts b/x-pack/plugins/ingest_manager/common/services/decode_cloud_id.ts
@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+// decodeCloudId decodes the c.id into c.esURL and c.kibURL
+export function decodeCloudId(
+  cid: string
+):
+  | {
+      host: string;
+      defaultPort: string;
+      elasticsearchUrl: string;
+      kibanaUrl: string;
+    }
+  | undefined {
+  // 1. Ignore anything before `:`.
+  const id = cid.split(':').pop();
+  if (!id) {
+    // throw new Error(`Unable to decode ${id}`);
+    // eslint-disable-next-line no-console
+    console.debug(`Unable to decode ${id}`);
+    return;
+  }
+
+  // 2. base64 decode
+  let decoded: string | undefined;
+  try {
+    decoded = Buffer.from(id, 'base64').toString('utf8');
+  } catch {
+    // throw new Error(`base64 decoding failed on ${id}`);
+    // eslint-disable-next-line no-console
+    console.debug(`base64 decoding failed on ${id}`);
+    return;
+  }
+
+  // 3. separate based on `$`
+  const words = decoded.split('$');
+  if (words.length < 3) {
+    // throw new Error(`Expected at least 3 parts in ${decoded}`);
+    // eslint-disable-next-line no-console
+    console.debug(`Expected at least 3 parts in ${decoded}`);
+    return;
+  }
+  // 4. extract port from the ES and Kibana host
+  const [host, defaultPort] = extractPortFromName(words[0]);
+  const [esId, esPort] = extractPortFromName(words[1], defaultPort);
+  const [kbId, kbPort] = extractPortFromName(words[2], defaultPort);
+  // 5. form the URLs
+  const esUrl = `https://${esId}.${host}:${esPort}`;
+  const kbUrl = `https://${kbId}.${host}:${kbPort}`;
+  return {
+    host,
+    defaultPort,
+    elasticsearchUrl: esUrl,
+    kibanaUrl: kbUrl,
+  };
+}
+// extractPortFromName takes a string in the form `id:port` and returns the
+// Id and the port. If there's no `:`, the default port is returned
+function extractPortFromName(word: string, defaultPort = '443') {
+  const [host, port = defaultPort] = word.split(':');
+  return [host, port];
+}
diff --git a/x-pack/plugins/ingest_manager/common/services/index.ts b/x-pack/plugins/ingest_manager/common/services/index.ts
@@ -9,3 +9,4 @@ export * from './routes';
 export { packageToConfigDatasourceInputs, packageToConfigDatasource } from './package_to_config';
 export { storedDatasourceToAgentDatasource } from './datasource_to_agent_datasource';
 export { AgentStatusKueryHelper };
+export { decodeCloudId } from './decode_cloud_id';
diff --git a/x-pack/plugins/ingest_manager/common/types/index.ts b/x-pack/plugins/ingest_manager/common/types/index.ts
@@ -15,7 +15,6 @@ export interface IngestManagerConfigType {
   fleet: {
     enabled: boolean;
     tlsCheckDisabled: boolean;
-    defaultOutputHost: string;
     kibana: {
       host?: string;
       ca_sha256?: string;

diff --git a/x-pack/plugins/ingest_manager/server/index.ts b/x-pack/plugins/ingest_manager/server/index.ts
@@ -32,7 +32,7 @@ export const config = {
         ca_sha256: schema.maybe(schema.string()),
       }),
       elasticsearch: schema.object({
-        host: schema.string({ defaultValue: 'http://localhost:9200' }),
+        host: schema.maybe(schema.string()),
         ca_sha256: schema.maybe(schema.string()),
       }),
     }),

diff --git a/x-pack/plugins/ingest_manager/server/services/output.ts b/x-pack/plugins/ingest_manager/server/services/output.ts
@@ -7,6 +7,7 @@ import { SavedObjectsClientContract } from 'src/core/server';
 import { NewOutput, Output } from '../types';
 import { DEFAULT_OUTPUT, OUTPUT_SAVED_OBJECT_TYPE } from '../constants';
 import { appContextService } from './app_context';
+import { decodeCloudId } from '../../common';
 
 const SAVED_OBJECT_TYPE = OUTPUT_SAVED_OBJECT_TYPE;
 
@@ -16,11 +17,17 @@ class OutputService {
       type: OUTPUT_SAVED_OBJECT_TYPE,
       filter: `${OUTPUT_SAVED_OBJECT_TYPE}.attributes.is_default:true`,
     });
+    const cloud = appContextService.getCloud();
+    const cloudId = cloud?.isCloudEnabled && cloud.cloudId;
+    const cloudUrl = cloudId && decodeCloudId(cloudId)?.elasticsearchUrl;
+    const flagsUrl = appContextService.getConfig()!.fleet.elasticsearch.host;
+    const defaultUrl = 'http://localhost:9200';
+    const defaultOutputUrl = cloudUrl || flagsUrl || defaultUrl;
 
     if (!outputs.saved_objects.length) {
       const newDefaultOutput = {
         ...DEFAULT_OUTPUT,
-        hosts: [appContextService.getConfig()!.fleet.elasticsearch.host],
+        hosts: [defaultOutputUrl],
         ca_sha256: appContextService.getConfig()!.fleet.elasticsearch.ca_sha256,
       } as NewOutput;
 

diff --git a/x-pack/plugins/ingest_manager/server/services/setup.ts b/x-pack/plugins/ingest_manager/server/services/setup.ts
@@ -18,6 +18,7 @@ import {
   Installation,
   Output,
   DEFAULT_AGENT_CONFIGS_PACKAGES,
+  decodeCloudId,
 } from '../../common';
 import { getPackageInfo } from './epm/packages';
 import { datasourceService } from './datasource';
@@ -43,7 +44,11 @@ export async function setupIngestManager(
         const serverInfo = http.getServerInfo();
         const basePath = http.basePath;
 
-        const defaultKibanaUrl = url.format({
+        const cloud = appContextService.getCloud();
+        const cloudId = cloud?.isCloudEnabled && cloud.cloudId;
+        const cloudUrl = cloudId && decodeCloudId(cloudId)?.kibanaUrl;
+        const flagsUrl = appContextService.getConfig()?.fleet?.kibana?.host;
+        const defaultUrl = url.format({
           protocol: serverInfo.protocol,
           hostname: serverInfo.host,
           port: serverInfo.port,
@@ -53,7 +58,7 @@ export async function setupIngestManager(
         return settingsService.saveSettings(soClient, {
           agent_auto_upgrade: true,
           package_auto_upgrade: true,
-          kibana_url: appContextService.getConfig()?.fleet?.kibana?.host ?? defaultKibanaUrl,
+          kibana_url: cloudUrl || flagsUrl || defaultUrl,
         });
       }