[Security Solution][Endpoint] Add Serverless support to data loading …

…utilities (#166402) ## Summary PR enables the existing data loading utilities/services, used in e2e testing and CLI tools, to support being run against a serverless Env.. Changes include: - `createRuntimeServices()` and the associated methods that create the ES and KBN clients, will now by default add a CA cert to the ES and KBN clients if the URL protocol is `https` - an option was also added to the mothods that allows a developer to turn this behaviour off if necessary (`noCertForSsl`) - `createRuntimeServices()` option `asSuperuser` will NOT attempt to create a new user in ES if it detects its running against serverless. It will instead set the `username` to `system_indices_superuser` - `resolver_generator.js` script was updated so that it can be run against a serverless env. (note: tested only in local dev, not agains cloud environments) - new utility to determine if Kibana is running in serverless mode (`isServerlessKibanaFlavor()`) - Cypress tests that don't require specific user/role were updated to use `system_indices_superuser` as the default username (instead of `elastic`)
elastic · Sep 14, 2023 · a449481 · a449481
1 parent ef020b2
commit a449481
Show file tree

Hide file tree

Showing 10 changed files with 201 additions and 93 deletions.
diff --git a/x-pack/plugins/security_solution/common/endpoint/data_loaders/index_endpoint_rule_alerts.ts b/x-pack/plugins/security_solution/common/endpoint/data_loaders/index_endpoint_rule_alerts.ts
@@ -5488,10 +5488,6 @@ const getAlertsIndexMappings = (): IndexMappings => {
         index: {
           auto_expand_replicas: '0-1',
           hidden: 'true',
-          lifecycle: {
-            name: '.alerts-ilm-policy',
-            rollover_alias: '.alerts-security.alerts-default',
-          },
           mapping: {
             total_fields: {
               limit: 1900,

diff --git a/x-pack/plugins/security_solution/common/endpoint/data_loaders/setup_fleet_for_endpoint.ts b/x-pack/plugins/security_solution/common/endpoint/data_loaders/setup_fleet_for_endpoint.ts
@@ -21,7 +21,12 @@ import {
 } from '@kbn/fleet-plugin/common';
 import { ToolingLog } from '@kbn/tooling-log';
 import { UsageTracker } from './usage_tracker';
-import { EndpointDataLoadingError, retryOnError, wrapErrorAndRejectPromise } from './utils';
+import {
+  EndpointDataLoadingError,
+  RETRYABLE_TRANSIENT_ERRORS,
+  retryOnError,
+  wrapErrorAndRejectPromise,
+} from './utils';
 
 const usageTracker = new UsageTracker({ dumpOnProcessExit: true });
 
@@ -165,13 +170,7 @@ export const installOrUpgradeEndpointFleetPackage = async (
     return bulkResp[0] as BulkInstallPackageInfo;
   };
 
-  return retryOnError(
-    updatePackages,
-    ['no_shard_available_action_exception', 'illegal_index_shard_state_exception'],
-    logger,
-    5,
-    10000
-  )
+  return retryOnError(updatePackages, RETRYABLE_TRANSIENT_ERRORS, logger, 5, 10000)
     .then((result) => {
       usageRecord.set('success');
 

diff --git a/x-pack/plugins/security_solution/common/endpoint/data_loaders/utils.ts b/x-pack/plugins/security_solution/common/endpoint/data_loaders/utils.ts
@@ -8,6 +8,11 @@
 import { mergeWith } from 'lodash';
 import { ToolingLog } from '@kbn/tooling-log';
 
+export const RETRYABLE_TRANSIENT_ERRORS: Readonly<Array<string | RegExp>> = [
+  'no_shard_available_action_exception',
+  'illegal_index_shard_state_exception',
+];
+
 export class EndpointDataLoadingError extends Error {
   constructor(message: string, public meta?: unknown) {
     super(message);
@@ -43,7 +48,7 @@ export const mergeAndAppendArrays = <T, S>(destinationObj: T, srcObj: S): T => {
  */
 export const retryOnError = async <T>(
   callback: () => Promise<T>,
-  errors: Array<string | RegExp>,
+  errors: Array<string | RegExp> | Readonly<Array<string | RegExp>>,
   logger?: ToolingLog,
   tryCount: number = 5,
   interval: number = 10000
@@ -60,6 +65,8 @@ export const retryOnError = async <T>(
     });
   };
 
+  log.indent(4);
+
   let attempt = 1;
   let responsePromise: Promise<T>;
 
@@ -71,20 +78,31 @@ export const retryOnError = async <T>(
 
     try {
       responsePromise = callback(); // store promise so that if it fails and no more attempts, we return the last failure
-      return await responsePromise;
+      const result = await responsePromise;
+
+      log.info(msg(`attempt ${thisAttempt} was successful. Exiting retry`));
+      log.indent(-4);
+
+      return result;
     } catch (err) {
       log.info(msg(`attempt ${thisAttempt} failed with: ${err.message}`), err);
 
       // If not an error that is retryable, then end loop here and return that error;
       if (!isRetryableError(err)) {
         log.error(err);
+        log.error(msg('non-retryable error encountered'));
+        log.indent(-4);
         return Promise.reject(err);
       }
     }
 
     await new Promise((resolve) => setTimeout(resolve, interval));
   }
 
+  log.error(msg(`max retry attempts reached. returning last failure`));
+  log.indent(-4);
+
+  // Last resort: return the last rejected Promise.
   // @ts-expect-error TS2454: Variable 'responsePromise' is used before being assigned.
   return responsePromise;
 };
diff --git a/x-pack/plugins/security_solution/public/management/cypress.config.ts b/x-pack/plugins/security_solution/public/management/cypress.config.ts
@@ -37,7 +37,7 @@ export default defineCypressConfig({
     ELASTICSEARCH_URL: 'http://localhost:9200',
     FLEET_SERVER_URL: 'https://localhost:8220',
     // Username/password used for both elastic and kibana
-    KIBANA_USERNAME: 'elastic',
+    KIBANA_USERNAME: 'system_indices_superuser',
     KIBANA_PASSWORD: 'changeme',
     ELASTICSEARCH_USERNAME: 'system_indices_superuser',
     ELASTICSEARCH_PASSWORD: 'changeme',

diff --git a/x-pack/plugins/security_solution/public/management/cypress_endpoint.config.ts b/x-pack/plugins/security_solution/public/management/cypress_endpoint.config.ts
@@ -39,6 +39,10 @@ export default defineCypressConfig({
     'cypress-react-selector': {
       root: '#security-solution-app',
     },
+    KIBANA_USERNAME: 'system_indices_superuser',
+    KIBANA_PASSWORD: 'changeme',
+    ELASTICSEARCH_USERNAME: 'system_indices_superuser',
+    ELASTICSEARCH_PASSWORD: 'changeme',
   },
 
   e2e: {

diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts
@@ -37,6 +37,10 @@ import type {
 import nodeFetch from 'node-fetch';
 import semver from 'semver';
 import axios from 'axios';
+import {
+  RETRYABLE_TRANSIENT_ERRORS,
+  retryOnError,
+} from '../../../common/endpoint/data_loaders/utils';
 import { fetchKibanaStatus } from './stack_services';
 import { catchAxiosErrorFormatAndThrow } from './format_axios_error';
 import { FleetAgentGenerator } from '../../../common/endpoint/data_generators/fleet_agent_generator';
@@ -137,11 +141,15 @@ export const waitForHostToEnroll = async (
   let found: Agent | undefined;
 
   while (!found && !hasTimedOut()) {
-    found = await fetchFleetAgents(kbnClient, {
-      perPage: 1,
-      kuery: `(local_metadata.host.hostname.keyword : "${hostname}") and (status:online)`,
-      showInactive: false,
-    }).then((response) => response.items[0]);
+    found = await retryOnError(
+      async () =>
+        fetchFleetAgents(kbnClient, {
+          perPage: 1,
+          kuery: `(local_metadata.host.hostname.keyword : "${hostname}") and (status:online)`,
+          showInactive: false,
+        }).then((response) => response.items[0]),
+      RETRYABLE_TRANSIENT_ERRORS
+    );
 
     if (!found) {
       // sleep and check again

diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/format_axios_error.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/format_axios_error.ts
@@ -22,7 +22,11 @@ export class FormattedAxiosError extends Error {
   };
 
   constructor(axiosError: AxiosError) {
-    super(axiosError.message);
+    super(
+      `${axiosError.message}${
+        axiosError?.response?.data ? `: ${JSON.stringify(axiosError?.response?.data)}` : ''
+      }`
+    );
 
     this.request = {
       method: axiosError.config?.method ?? '?',

diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/stack_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/stack_services.ts
@@ -15,11 +15,15 @@ import nodeFetch from 'node-fetch';
 import type { ReqOptions } from '@kbn/test/src/kbn_client/kbn_client_requester';
 import { type AxiosResponse } from 'axios';
 import type { ClientOptions } from '@elastic/elasticsearch/lib/client';
+import fs from 'fs';
+import { CA_CERT_PATH } from '@kbn/dev-utils';
 import { catchAxiosErrorFormatAndThrow } from './format_axios_error';
 import { isLocalhost } from './is_localhost';
 import { getLocalhostRealIp } from './localhost_services';
 import { createSecuritySuperuser } from './security_user_services';
 
+const CA_CERTIFICATE: Buffer = fs.readFileSync(CA_CERT_PATH);
+
 export interface RuntimeServices {
   kbnClient: KbnClient;
   esClient: Client;
@@ -64,6 +68,8 @@ interface CreateRuntimeServicesOptions {
   esPassword?: string;
   log?: ToolingLog;
   asSuperuser?: boolean;
+  /** If true, then a certificate will not be used when creating the Kbn/Es clients when url is `https` */
+  noCertForSsl?: boolean;
 }
 
 class KbnClientExtended extends KbnClient {
@@ -105,26 +111,39 @@ export const createRuntimeServices = async ({
   esPassword,
   log = new ToolingLog({ level: 'info', writeTo: process.stdout }),
   asSuperuser = false,
+  noCertForSsl,
 }: CreateRuntimeServicesOptions): Promise<RuntimeServices> => {
   let username = _username;
   let password = _password;
 
   if (asSuperuser) {
     await waitForKibana(kibanaUrl);
+    const tmpEsClient = createEsClient({
+      url: elasticsearchUrl,
+      username,
+      password,
+      log,
+      noCertForSsl,
+    });
 
-    const superuserResponse = await createSecuritySuperuser(
-      createEsClient({
-        url: elasticsearchUrl,
-        username,
-        password,
-        log,
-      })
-    );
+    const isServerlessEs = (await tmpEsClient.info()).version.build_flavor === 'serverless';
+
+    if (isServerlessEs) {
+      log?.warning(
+        'Creating Security Superuser is not supported in current environment. ES is running in serverless mode. ' +
+          'Will use username [system_indices_superuser] instead.'
+      );
+
+      username = 'system_indices_superuser';
+      password = 'changeme';
+    } else {
+      const superuserResponse = await createSecuritySuperuser(tmpEsClient);
 
-    ({ username, password } = superuserResponse);
+      ({ username, password } = superuserResponse);
 
-    if (superuserResponse.created) {
-      log.info(`Kibana user [${username}] was crated with password [${password}]`);
+      if (superuserResponse.created) {
+        log.info(`Kibana user [${username}] was crated with password [${password}]`);
+      }
     }
   }
 
@@ -133,16 +152,17 @@ export const createRuntimeServices = async ({
   const fleetURL = new URL(fleetServerUrl);
 
   return {
-    kbnClient: createKbnClient({ log, url: kibanaUrl, username, password, apiKey }),
+    kbnClient: createKbnClient({ log, url: kibanaUrl, username, password, apiKey, noCertForSsl }),
     esClient: createEsClient({
       log,
       url: elasticsearchUrl,
       username: esUsername ?? username,
       password: esPassword ?? password,
       apiKey,
+      noCertForSsl,
     }),
     log,
-    localhostRealIp: await getLocalhostRealIp(),
+    localhostRealIp: getLocalhostRealIp(),
     apiKey: apiKey ?? '',
     user: {
       username,
@@ -188,18 +208,27 @@ export const createEsClient = ({
   password,
   apiKey,
   log,
+  noCertForSsl,
 }: {
   url: string;
   username: string;
   password: string;
   /** If defined, both `username` and `password` will be ignored */
   apiKey?: string;
   log?: ToolingLog;
+  noCertForSsl?: boolean;
 }): Client => {
+  const isHttps = new URL(url).protocol.startsWith('https');
   const clientOptions: ClientOptions = {
     node: buildUrlWithCredentials(url, apiKey ? '' : username, apiKey ? '' : password),
   };
 
+  if (isHttps && !noCertForSsl) {
+    clientOptions.tls = {
+      ca: [CA_CERTIFICATE],
+    };
+  }
+
   if (apiKey) {
     clientOptions.auth = { apiKey };
   }
@@ -217,23 +246,36 @@ export const createKbnClient = ({
   password,
   apiKey,
   log = new ToolingLog(),
+  noCertForSsl,
 }: {
   url: string;
   username: string;
   password: string;
   /** If defined, both `username` and `password` will be ignored */
   apiKey?: string;
   log?: ToolingLog;
+  noCertForSsl?: boolean;
 }): KbnClient => {
-  const kbnUrl = buildUrlWithCredentials(url, username, password);
+  const isHttps = new URL(url).protocol.startsWith('https');
+  const clientOptions: ConstructorParameters<typeof KbnClientExtended>[0] = {
+    log,
+    apiKey,
+    url: buildUrlWithCredentials(url, username, password),
+  };
+
+  if (isHttps && !noCertForSsl) {
+    clientOptions.certificateAuthorities = [CA_CERTIFICATE];
+  }
 
   if (log) {
     log.verbose(
-      `Creating Kibana client with URL: ${kbnUrl} ${apiKey ? ` + ApiKey: ${apiKey}` : ''}`
+      `Creating Kibana client with URL: ${clientOptions.url} ${
+        apiKey ? ` + ApiKey: ${apiKey}` : ''
+      }`
     );
   }
 
-  return new KbnClientExtended({ log, url: kbnUrl, apiKey });
+  return new KbnClientExtended(clientOptions);
 };
 
 /**
@@ -287,3 +329,18 @@ export const waitForKibana = async (kbnUrl: string): Promise<void> => {
     { maxTimeout: 10000 }
   );
 };
+
+export const isServerlessKibanaFlavor = async (kbnClient: KbnClient): Promise<boolean> => {
+  const kbnStatus = await fetchKibanaStatus(kbnClient);
+
+  // If we don't have status for plugins, then error
+  // the Status API will always return something (its an open API), but if auth was successful,
+  // it will also return more data.
+  if (!kbnStatus.status.plugins) {
+    throw new Error(
+      `Unable to retrieve Kibana plugins status (likely an auth issue with the username being used for kibana)`
+    );
+  }
+
+  return kbnStatus.status.plugins?.serverless?.level === 'available';
+};