Merge branch 'main' into carlosdelest/sparse_vector_field_support

.buildkite/pipelines/pull_request/storybooks.yml

@@ -2,6 +2,6 @@ steps:
   - command: .buildkite/scripts/steps/storybooks/build_and_upload.sh
     label: 'Build Storybooks'
     agents:
-      queue: n2-4-spot
+      queue: n2-8-spot
     key: storybooks
     timeout_in_minutes: 60

src/dev/performance/run_performance_cli.ts

@@ -11,6 +11,7 @@ import { run } from '@kbn/dev-cli-runner';
 import { REPO_ROOT } from '@kbn/repo-info';
 import fs from 'fs';
 import path from 'path';
+import { JOURNEY_APM_CONFIG } from '@kbn/journeys';
 
 const JOURNEY_BASE_PATH = 'x-pack/performance/journeys';
 
@@ -107,21 +108,20 @@ run(
           'scripts/es',
           'snapshot',
           '--license=trial',
-          // Disabling APM until https://github.com/elastic/elasticsearch/issues/100072 is fixed
-          // ...(JOURNEY_APM_CONFIG.active
-          //   ? [
-          //       '-E',
-          //       'tracing.apm.enabled=true',
-          //       '-E',
-          //       'tracing.apm.agent.transaction_sample_rate=1.0',
-          //       '-E',
-          //       `tracing.apm.agent.server_url=${JOURNEY_APM_CONFIG.serverUrl}`,
-          //       '-E',
-          //       `tracing.apm.agent.secret_token=${JOURNEY_APM_CONFIG.secretToken}`,
-          //       '-E',
-          //       `tracing.apm.agent.environment=${JOURNEY_APM_CONFIG.environment}`,
-          //     ]
-          //   : []),
+          ...(JOURNEY_APM_CONFIG.active
+            ? [
+                '-E',
+                'tracing.apm.enabled=true',
+                '-E',
+                'tracing.apm.agent.transaction_sample_rate=1.0',
+                '-E',
+                `tracing.apm.agent.server_url=${JOURNEY_APM_CONFIG.serverUrl}`,
+                '-E',
+                `tracing.apm.agent.secret_token=${JOURNEY_APM_CONFIG.secretToken}`,
+                '-E',
+                `tracing.apm.agent.environment=${JOURNEY_APM_CONFIG.environment}`,
+              ]
+            : []),
         ],
         cwd: REPO_ROOT,
         wait: /kbn\/es setup complete/,

src/dev/tsconfig.json

@@ -39,5 +39,6 @@
     "@kbn/get-repo-files",
     "@kbn/import-locator",
     "@kbn/config-schema",
+    "@kbn/journeys",
   ]
 }

test/functional/apps/discover/group1/_shared_links.ts

@@ -22,8 +22,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const toasts = getService('toasts');
   const deployment = getService('deployment');
 
-  // Failing: See https://github.com/elastic/kibana/issues/167405
-  describe.skip('shared links', function describeIndexTests() {
+  describe('shared links', function describeIndexTests() {
     let baseUrl: string;
 
     async function setup({ storeStateInSessionStorage }: { storeStateInSessionStorage: boolean }) {
@@ -174,6 +173,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
           const resolvedTime = await PageObjects.timePicker.getTimeConfig();
           expect(resolvedTime.start).to.equal(actualTime.start);
           expect(resolvedTime.end).to.equal(actualTime.end);
+          await toasts.dismissAllToasts();
           return true;
         });
       });

x-pack/plugins/log_explorer/public/services/datasets/datasets_client.ts

@@ -7,6 +7,7 @@
 
 import { HttpStart } from '@kbn/core/public';
 
+import { API_VERSIONS } from '@kbn/fleet-plugin/common';
 import { Dataset, Integration } from '../../../common/datasets';
 import {
   DATASETS_URL,
@@ -43,9 +44,11 @@ export class DatasetsClient implements IDatasetsClient {
 
     const query = findIntegrationsRequestQueryRT.encode(search);
 
-    const response = await this.http.get(INTEGRATIONS_URL, { query }).catch((error) => {
-      throw new FindIntegrationsError(`Failed to fetch integrations": ${error}`);
-    });
+    const response = await this.http
+      .get(INTEGRATIONS_URL, { query, version: API_VERSIONS.public.v1 })
+      .catch((error) => {
+        throw new FindIntegrationsError(`Failed to fetch integrations": ${error}`);
+      });
 
     const data = decodeOrThrow(
       findIntegrationsResponseRT,
@@ -61,9 +64,11 @@ export class DatasetsClient implements IDatasetsClient {
 
     const query = findDatasetsRequestQueryRT.encode(search);
 
-    const response = await this.http.get(DATASETS_URL, { query }).catch((error) => {
-      throw new FindDatasetsError(`Failed to fetch data streams": ${error}`);
-    });
+    const response = await this.http
+      .get(DATASETS_URL, { query, version: API_VERSIONS.public.v1 })
+      .catch((error) => {
+        throw new FindDatasetsError(`Failed to fetch data streams": ${error}`);
+      });
 
     const data = decodeOrThrow(
       findDatasetsResponseRT,

x-pack/plugins/observability/server/services/slo/update_slo.test.ts

@@ -45,7 +45,7 @@ describe('UpdateSLO', () => {
     const newSettings = { ...slo.settings, timestamp_field: 'newField' };
     await updateSLO.execute(slo.id, { settings: newSettings });
 
-    expectDeletionOfObsoleteSLOData(slo);
+    expectDeletionOfOriginalSLO(slo);
     expect(mockRepository.save).toBeCalledWith(
       expect.objectContaining({
         ...slo,
@@ -70,7 +70,7 @@ describe('UpdateSLO', () => {
       },
     });
 
-    expectDeletionOfObsoleteSLOData(slo);
+    expectDeletionOfOriginalSLO(slo);
     expectInstallationOfNewSLOTransform();
   });
 
@@ -86,7 +86,7 @@ describe('UpdateSLO', () => {
       },
     });
 
-    expectDeletionOfObsoleteSLOData(slo);
+    expectDeletionOfOriginalSLO(slo);
     expectInstallationOfNewSLOTransform();
   });
 
@@ -102,7 +102,7 @@ describe('UpdateSLO', () => {
       },
     });
 
-    expectDeletionOfObsoleteSLOData(slo);
+    expectDeletionOfOriginalSLO(slo);
     expectInstallationOfNewSLOTransform();
   });
 
@@ -119,7 +119,7 @@ describe('UpdateSLO', () => {
     expect(mockEsClient.index.mock.calls[0]).toMatchSnapshot();
   });
 
-  it('removes the obsolete data from the SLO previous revision', async () => {
+  it('removes the original data from the original SLO', async () => {
     const slo = createSLO({
       indicator: createAPMTransactionErrorRateIndicator({ environment: 'development' }),
     });
@@ -128,7 +128,6 @@ describe('UpdateSLO', () => {
     const newIndicator = createAPMTransactionErrorRateIndicator({ environment: 'production' });
     await updateSLO.execute(slo.id, { indicator: newIndicator });
 
-    expectDeletionOfObsoleteSLOData(slo);
     expect(mockRepository.save).toBeCalledWith(
       expect.objectContaining({
         ...slo,
@@ -138,6 +137,53 @@ describe('UpdateSLO', () => {
       })
     );
     expectInstallationOfNewSLOTransform();
+    expectDeletionOfOriginalSLO(slo);
+  });
+
+  describe('when error happens during the transform installation step', () => {
+    it('restores the previous SLO definition in the repository', async () => {
+      const slo = createSLO({
+        indicator: createAPMTransactionErrorRateIndicator({ environment: 'development' }),
+      });
+      mockRepository.findById.mockResolvedValueOnce(slo);
+      mockTransformManager.install.mockRejectedValueOnce(new Error('Transform install error'));
+
+      const newIndicator = createAPMTransactionErrorRateIndicator({ environment: 'production' });
+
+      await expect(updateSLO.execute(slo.id, { indicator: newIndicator })).rejects.toThrowError(
+        'Transform install error'
+      );
+
+      expect(mockRepository.save).toHaveBeenCalledWith(slo);
+      expect(mockTransformManager.preview).not.toHaveBeenCalled();
+      expect(mockTransformManager.start).not.toHaveBeenCalled();
+      expect(mockTransformManager.stop).not.toHaveBeenCalled();
+      expect(mockTransformManager.uninstall).not.toHaveBeenCalled();
+      expect(mockEsClient.deleteByQuery).not.toHaveBeenCalled();
+    });
+  });
+
+  describe('when error happens during the transform start step', () => {
+    it('removes the new transform and restores the previous SLO definition in the repository', async () => {
+      const slo = createSLO({
+        indicator: createAPMTransactionErrorRateIndicator({ environment: 'development' }),
+      });
+      mockRepository.findById.mockResolvedValueOnce(slo);
+      mockTransformManager.start.mockRejectedValueOnce(new Error('Transform start error'));
+
+      const newIndicator = createAPMTransactionErrorRateIndicator({ environment: 'production' });
+
+      await expect(updateSLO.execute(slo.id, { indicator: newIndicator })).rejects.toThrowError(
+        'Transform start error'
+      );
+
+      expect(mockTransformManager.uninstall).toHaveBeenCalledWith(
+        getSLOTransformId(slo.id, slo.revision + 1)
+      );
+      expect(mockRepository.save).toHaveBeenCalledWith(slo);
+      expect(mockTransformManager.stop).not.toHaveBeenCalled();
+      expect(mockEsClient.deleteByQuery).not.toHaveBeenCalled();
+    });
   });
 
   function expectInstallationOfNewSLOTransform() {
@@ -146,12 +192,12 @@ describe('UpdateSLO', () => {
     expect(mockTransformManager.start).toBeCalled();
   }
 
-  function expectDeletionOfObsoleteSLOData(originalSlo: SLO) {
+  function expectDeletionOfOriginalSLO(originalSlo: SLO) {
     const transformId = getSLOTransformId(originalSlo.id, originalSlo.revision);
     expect(mockTransformManager.stop).toBeCalledWith(transformId);
     expect(mockTransformManager.uninstall).toBeCalledWith(transformId);
-    expect(mockEsClient.deleteByQuery).toHaveBeenCalledTimes(2);
 
+    expect(mockEsClient.deleteByQuery).toHaveBeenCalledTimes(2);
     expect(mockEsClient.deleteByQuery).toHaveBeenNthCalledWith(
       1,
       expect.objectContaining({

x-pack/plugins/observability/server/services/slo/update_slo.ts

@@ -36,13 +36,27 @@ export class UpdateSLO {
 
     validateSLO(updatedSlo);
 
-    await this.deleteObsoleteSLORevisionData(originalSlo);
-
     const updatedSloTransformId = getSLOTransformId(updatedSlo.id, updatedSlo.revision);
     await this.repository.save(updatedSlo);
-    await this.transformManager.install(updatedSlo);
-    await this.transformManager.preview(updatedSloTransformId);
-    await this.transformManager.start(updatedSloTransformId);
+
+    try {
+      await this.transformManager.install(updatedSlo);
+    } catch (err) {
+      await this.repository.save(originalSlo);
+      throw err;
+    }
+
+    try {
+      await this.transformManager.preview(updatedSloTransformId);
+      await this.transformManager.start(updatedSloTransformId);
+    } catch (err) {
+      await Promise.all([
+        this.transformManager.uninstall(updatedSloTransformId),
+        this.repository.save(originalSlo),
+      ]);
+
+      throw err;
+    }
 
     await this.esClient.index({
       index: SLO_SUMMARY_TEMP_INDEX_NAME,
@@ -51,13 +65,21 @@ export class UpdateSLO {
       refresh: true,
     });
 
+    await this.deleteOriginalSLO(originalSlo);
+
     return this.toResponse(updatedSlo);
   }
 
-  private async deleteObsoleteSLORevisionData(originalSlo: SLO) {
-    const originalSloTransformId = getSLOTransformId(originalSlo.id, originalSlo.revision);
-    await this.transformManager.stop(originalSloTransformId);
-    await this.transformManager.uninstall(originalSloTransformId);
+  private async deleteOriginalSLO(originalSlo: SLO) {
+    try {
+      const originalSloTransformId = getSLOTransformId(originalSlo.id, originalSlo.revision);
+      await this.transformManager.stop(originalSloTransformId);
+      await this.transformManager.uninstall(originalSloTransformId);
+    } catch (err) {
+      // Any errors here should not prevent moving forward.
+      // Worst case we keep rolling up data for the previous revision number.
+    }
+
     await this.deleteRollupData(originalSlo.id, originalSlo.revision);
     await this.deleteSummaryData(originalSlo.id, originalSlo.revision);
   }

x-pack/test/apm_api_integration/tests/alerts/anomaly_alert.spec.ts

@@ -17,10 +17,10 @@ import { waitForRuleStatus } from './helpers/wait_for_rule_status';
 
 export default function ApiTest({ getService }: FtrProviderContext) {
   const registry = getService('registry');
-
   const supertest = getService('supertest');
   const ml = getService('ml');
   const es = getService('es');
+  const logger = getService('log');
 
   const synthtraceEsClient = getService('synthtraceEsClient');
   // FLAKY https://github.com/elastic/kibana/issues/160298
@@ -68,8 +68,12 @@ export default function ApiTest({ getService }: FtrProviderContext) {
       });
 
       after(async () => {
-        await synthtraceEsClient.clean();
-        await deleteRuleById({ supertest, ruleId });
+        try {
+          await synthtraceEsClient.clean();
+          await deleteRuleById({ supertest, ruleId });
+        } catch (e) {
+          logger.info('Could not delete rule by id', e);
+        }
       });
 
       describe('with ml jobs', () => {

x-pack/test/apm_api_integration/tests/alerts/error_count_threshold.spec.ts

@@ -289,8 +289,12 @@ export default function ApiTest({ getService }: FtrProviderContext) {
       });
 
       after(async () => {
-        await deleteRuleById({ supertest, ruleId });
-        await deleteAlertsByRuleId({ es, ruleId });
+        try {
+          await deleteRuleById({ supertest, ruleId });
+          await deleteAlertsByRuleId({ es, ruleId });
+        } catch (e) {
+          logger.info('Could not delete rule', e);
+        }
       });
 
       it('produces one alert for the opbeans-php service', async () => {

x-pack/test/apm_api_integration/tests/alerts/transaction_duration.spec.ts

@@ -77,8 +77,12 @@ export default function ApiTest({ getService }: FtrProviderContext) {
     });
 
     after(async () => {
-      await synthtraceEsClient.clean();
-      await clearKibanaApmEventLog(es);
+      try {
+        await synthtraceEsClient.clean();
+        await clearKibanaApmEventLog(es);
+      } catch (e) {
+        logger.info('Could not clear apm event log', e);
+      }
     });
 
     describe('create rule for opbeans-java without kql filter', () => {
@@ -229,8 +233,12 @@ export default function ApiTest({ getService }: FtrProviderContext) {
       });
 
       after(async () => {
-        await deleteAlertsByRuleId({ es, ruleId });
-        await deleteRuleById({ supertest, ruleId });
+        try {
+          await deleteAlertsByRuleId({ es, ruleId });
+          await deleteRuleById({ supertest, ruleId });
+        } catch (e) {
+          logger.info('Could not delete rule or action connector', e);
+        }
       });
 
       it('checks if rule is active', async () => {

x-pack/test/apm_api_integration/tests/alerts/transaction_error_rate.spec.ts

@@ -76,8 +76,12 @@ export default function ApiTest({ getService }: FtrProviderContext) {
     });
 
     after(async () => {
-      await synthtraceEsClient.clean();
-      await clearKibanaApmEventLog(es);
+      try {
+        await synthtraceEsClient.clean();
+        await clearKibanaApmEventLog(es);
+      } catch (e) {
+        logger.info('Could not clean up apm event log', e);
+      }
     });
 
     describe('create rule without kql query', () => {
@@ -250,8 +254,12 @@ export default function ApiTest({ getService }: FtrProviderContext) {
       });
 
       after(async () => {
-        await deleteRuleById({ supertest, ruleId });
-        await deleteAlertsByRuleId({ es, ruleId });
+        try {
+          await deleteRuleById({ supertest, ruleId });
+          await deleteAlertsByRuleId({ es, ruleId });
+        } catch (e) {
+          logger.info('Could not delete rule', e);
+        }
       });
 
       it('indexes alert document with all group-by fields', async () => {