[8.13] [Fleet] Add retry logic to serverless API check (#176808) (#17…

…8508)

# Backport

This will backport the following commits from `main` to `8.13`:
- [[Fleet] Add retry logic to serverless API check
(#176808)](#176808)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Jill
Guyonnet","email":"jill.guyonnet@elastic.co"},"sourceCommit":{"committedDate":"2024-02-21T09:14:10Z","message":"[Fleet]
Add retry logic to serverless API check (#176808)\n\n##
Summary\r\n\r\nCloses
#176352
#176399
added the possibility to\r\nconfigure new Fleet Server hosts in
serverless, with the constraint that\r\nthe host URL must match the
default URL. The API integration tests\r\nwritten to test this have been
flaky, due to failure retrieving the\r\ndefault Fleet Server host or
default Elasticsearch output from saved\r\nobjects. This PR adds retry
in the tests.\r\n\r\nNote: I have tried adding retry logic in the API
handlers but kept\r\nhitting test flakiness.\r\n\r\nThis fix has been
tested using the Flaky Test Runner Pipeline, with\r\n48/49 test runs for
observability and security project
types:\r\n🟢\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/5218\r\n🟢\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/5222\r\n🟢\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/5225\r\n\r\n###
Checklist\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common scenarios\r\n- [x] [Flaky
Test\r\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\r\nused on any tests
changed","sha":"83e64be5d6730ec1f22ade806192bf1eab54751a","branchLabelMapping":{"^v8.14.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","Team:Fleet","backport:prev-minor","v8.14.0"],"number":176808,"url":"#176808
Add retry logic to serverless API check (#176808)\n\n##
Summary\r\n\r\nCloses
#176352
#176399
added the possibility to\r\nconfigure new Fleet Server hosts in
serverless, with the constraint that\r\nthe host URL must match the
default URL. The API integration tests\r\nwritten to test this have been
flaky, due to failure retrieving the\r\ndefault Fleet Server host or
default Elasticsearch output from saved\r\nobjects. This PR adds retry
in the tests.\r\n\r\nNote: I have tried adding retry logic in the API
handlers but kept\r\nhitting test flakiness.\r\n\r\nThis fix has been
tested using the Flaky Test Runner Pipeline, with\r\n48/49 test runs for
observability and security project
types:\r\n🟢\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/5218\r\n🟢\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/5222\r\n🟢\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/5225\r\n\r\n###
Checklist\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common scenarios\r\n- [x] [Flaky
Test\r\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\r\nused on any tests
changed","sha":"83e64be5d6730ec1f22ade806192bf1eab54751a"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v8.14.0","labelRegex":"^v8.14.0$","isSourceBranch":true,"state":"MERGED","url":"#176808
Add retry logic to serverless API check (#176808)\n\n##
Summary\r\n\r\nCloses
#176352
#176399
added the possibility to\r\nconfigure new Fleet Server hosts in
serverless, with the constraint that\r\nthe host URL must match the
default URL. The API integration tests\r\nwritten to test this have been
flaky, due to failure retrieving the\r\ndefault Fleet Server host or
default Elasticsearch output from saved\r\nobjects. This PR adds retry
in the tests.\r\n\r\nNote: I have tried adding retry logic in the API
handlers but kept\r\nhitting test flakiness.\r\n\r\nThis fix has been
tested using the Flaky Test Runner Pipeline, with\r\n48/49 test runs for
observability and security project
types:\r\n🟢\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/5218\r\n🟢\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/5222\r\n🟢\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/5225\r\n\r\n###
Checklist\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common scenarios\r\n- [x] [Flaky
Test\r\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\r\nused on any tests
changed","sha":"83e64be5d6730ec1f22ade806192bf1eab54751a"}}]}]
BACKPORT-->

x-pack/plugins/fleet/server/routes/fleet_server_hosts/handler.ts

@@ -36,6 +36,7 @@ async function checkFleetServerHostsWriteAPIsAllowed(
     return;
   }
 
+  // Fleet Server hosts must have the default host URL in serverless.
   const serverlessDefaultFleetServerHost = await getFleetServerHost(
     soClient,
     SERVERLESS_DEFAULT_FLEET_SERVER_HOST_ID

x-pack/plugins/fleet/server/routes/output/handler.test.ts

@@ -144,11 +144,19 @@ describe('output handler', () => {
 
   it('should return error on put elasticsearch output in serverless if host url is different from default', async () => {
     jest.spyOn(appContextService, 'getCloud').mockReturnValue({ isServerlessEnabled: true } as any);
+    // The original output should provide the output type
+    jest.spyOn(outputService, 'get').mockImplementation((_, id: string) => {
+      if (id === SERVERLESS_DEFAULT_OUTPUT_ID) {
+        return { hosts: ['http://elasticsearch:9200'] } as any;
+      } else {
+        return { id: 'output1', type: 'elasticsearch' } as any;
+      }
+    });
 
     const res = await putOutputHandler(
       mockContext,
       {
-        body: { type: 'elasticsearch', hosts: ['http://localhost:8080'] },
+        body: { hosts: ['http://localhost:8080'] },
         params: { outputId: 'output1' },
       } as any,
       mockResponse as any
@@ -169,7 +177,7 @@ describe('output handler', () => {
     const res = await putOutputHandler(
       mockContext,
       {
-        body: { type: 'elasticsearch', hosts: ['http://elasticsearch:9200'] },
+        body: { hosts: ['http://elasticsearch:9200'] },
         params: { outputId: 'output1' },
       } as any,
       mockResponse as any
@@ -184,7 +192,7 @@ describe('output handler', () => {
     const res = await putOutputHandler(
       mockContext,
       {
-        body: { type: 'elasticsearch', name: 'Renamed output' },
+        body: { name: 'Renamed output' },
         params: { outputId: 'output1' },
       } as any,
       mockResponse as any
@@ -201,7 +209,7 @@ describe('output handler', () => {
     const res = await putOutputHandler(
       mockContext,
       {
-        body: { type: 'elasticsearch', hosts: ['http://localhost:8080'] },
+        body: { hosts: ['http://localhost:8080'] },
         params: { outputId: 'output1' },
       } as any,
       mockResponse as any

x-pack/test_serverless/api_integration/test_suites/common/fleet/default_setup.ts

@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../../ftr_provider_context';
+
+const defaultFleetServerHostId = 'default-fleet-server';
+const defaultFleetServerHostUrl = 'https://localhost:8220';
+const defaultElasticsearchOutputId = 'es-default-output';
+const defaultElasticsearchOutputHostUrl = 'https://localhost:9200';
+
+export async function expectDefaultFleetServer({ getService }: FtrProviderContext) {
+  const supertest = getService('supertest');
+  const retry = getService('retry');
+
+  await retry.waitForWithTimeout('get default fleet server', 30_000, async () => {
+    const { body, status } = await supertest.get(
+      `/api/fleet/fleet_server_hosts/${defaultFleetServerHostId}`
+    );
+    if (status === 200 && body.item.host_urls.includes(defaultFleetServerHostUrl)) {
+      return true;
+    } else {
+      throw new Error(`Expected default Fleet Server id ${defaultFleetServerHostId} to exist`);
+    }
+  });
+}
+
+export async function expectDefaultElasticsearchOutput({ getService }: FtrProviderContext) {
+  const supertest = getService('supertest');
+  const retry = getService('retry');
+
+  await retry.waitForWithTimeout('get default Elasticsearch output', 30_000, async () => {
+    const { body, status } = await supertest.get(
+      `/api/fleet/outputs/${defaultElasticsearchOutputId}`
+    );
+    if (status === 200 && body.item.hosts.includes(defaultElasticsearchOutputHostUrl)) {
+      return true;
+    } else {
+      throw new Error(
+        `Expected default Elasticsearch output id ${defaultElasticsearchOutputId} to exist`
+      );
+    }
+  });
+}
+
+export const kbnServerArgs = [
+  '--xpack.cloud.serverless.project_id=ftr_fake_project_id',
+  `--xpack.fleet.fleetServerHosts=[${JSON.stringify({
+    id: defaultFleetServerHostId,
+    name: 'Default Fleet Server',
+    is_default: true,
+    host_urls: [defaultFleetServerHostUrl],
+  })}]`,
+  `--xpack.fleet.outputs=[${JSON.stringify({
+    id: defaultElasticsearchOutputId,
+    name: 'Default Output',
+    type: 'elasticsearch',
+    is_default: true,
+    is_default_monitoring: true,
+    hosts: [defaultElasticsearchOutputHostUrl],
+  })}]`,
+];

x-pack/test_serverless/api_integration/test_suites/observability/fleet/config.ts

@@ -7,6 +7,7 @@
 
 import { createTestConfig } from '../../../config.base';
 import { services } from '../apm_api_integration/common/services';
+import { kbnServerArgs } from '../../common/fleet/default_setup';
 
 export default createTestConfig({
   serverlessProject: 'oblt',
@@ -21,21 +22,5 @@ export default createTestConfig({
   // https://github.com/elastic/project-controller/blob/main/internal/project/observability/config/elasticsearch.yml
   esServerArgs: ['xpack.ml.dfa.enabled=false', 'xpack.ml.nlp.enabled=false'],
 
-  kbnServerArgs: [
-    '--xpack.cloud.serverless.project_id=ftr_fake_project_id',
-    `--xpack.fleet.fleetServerHosts=[${JSON.stringify({
-      id: 'default-fleet-server',
-      name: 'Default Fleet Server',
-      is_default: true,
-      host_urls: ['https://localhost:8220'],
-    })}]`,
-    `--xpack.fleet.outputs=[${JSON.stringify({
-      id: 'es-default-output',
-      name: 'Default Output',
-      type: 'elasticsearch',
-      is_default: true,
-      is_default_monitoring: true,
-      hosts: ['https://localhost:9200'],
-    })}]`,
-  ],
+  kbnServerArgs,
 });

x-pack/test_serverless/api_integration/test_suites/observability/fleet/fleet.ts

@@ -7,14 +7,19 @@
 
 import expect from 'expect';
 import { FtrProviderContext } from '../../../ftr_provider_context';
+import {
+  expectDefaultElasticsearchOutput,
+  expectDefaultFleetServer,
+} from '../../common/fleet/default_setup';
 
-export default function ({ getService }: FtrProviderContext) {
-  const svlCommonApi = getService('svlCommonApi');
-  const supertest = getService('supertest');
+export default function (ctx: FtrProviderContext) {
+  const svlCommonApi = ctx.getService('svlCommonApi');
+  const supertest = ctx.getService('supertest');
 
-  // FLAKY: https://github.com/elastic/kibana/issues/176858
-  describe.skip('fleet', function () {
+  describe('fleet', function () {
     it('rejects request to create a new fleet server hosts if host url is different from default', async () => {
+      await expectDefaultFleetServer(ctx);
+
       const { body, status } = await supertest
         .post('/api/fleet/fleet_server_hosts')
         .set(svlCommonApi.getInternalRequestHeader())
@@ -33,6 +38,8 @@ export default function ({ getService }: FtrProviderContext) {
     });
 
     it('accepts request to create a new fleet server hosts if host url is same as default', async () => {
+      await expectDefaultFleetServer(ctx);
+
       const { body, status } = await supertest
         .post('/api/fleet/fleet_server_hosts')
         .set(svlCommonApi.getInternalRequestHeader())
@@ -51,6 +58,8 @@ export default function ({ getService }: FtrProviderContext) {
     });
 
     it('rejects request to create a new elasticsearch output if host is different from default', async () => {
+      await expectDefaultElasticsearchOutput(ctx);
+
       const { body, status } = await supertest
         .post('/api/fleet/outputs')
         .set(svlCommonApi.getInternalRequestHeader())
@@ -71,6 +80,8 @@ export default function ({ getService }: FtrProviderContext) {
     });
 
     it('accepts request to create a new elasticsearch output if host url is same as default', async () => {
+      await expectDefaultElasticsearchOutput(ctx);
+
       const { body, status } = await supertest
         .post('/api/fleet/outputs')
         .set(svlCommonApi.getInternalRequestHeader())

x-pack/test_serverless/api_integration/test_suites/security/fleet/config.ts

@@ -6,6 +6,7 @@
  */
 
 import { createTestConfig } from '../../../config.base';
+import { kbnServerArgs } from '../../common/fleet/default_setup';
 
 export default createTestConfig({
   serverlessProject: 'security',
@@ -19,21 +20,5 @@ export default createTestConfig({
   // https://github.com/elastic/project-controller/blob/main/internal/project/security/config/elasticsearch.yml
   esServerArgs: ['xpack.ml.nlp.enabled=false'],
 
-  kbnServerArgs: [
-    '--xpack.cloud.serverless.project_id=ftr_fake_project_id',
-    `--xpack.fleet.fleetServerHosts=[${JSON.stringify({
-      id: 'default-fleet-server',
-      name: 'Default Fleet Server',
-      is_default: true,
-      host_urls: ['https://localhost:8220'],
-    })}]`,
-    `--xpack.fleet.outputs=[${JSON.stringify({
-      id: 'es-default-output',
-      name: 'Default Output',
-      type: 'elasticsearch',
-      is_default: true,
-      is_default_monitoring: true,
-      hosts: ['https://localhost:9200'],
-    })}]`,
-  ],
+  kbnServerArgs,
 });

x-pack/test_serverless/api_integration/test_suites/security/fleet/fleet.ts

@@ -7,14 +7,19 @@
 
 import expect from 'expect';
 import { FtrProviderContext } from '../../../ftr_provider_context';
+import {
+  expectDefaultElasticsearchOutput,
+  expectDefaultFleetServer,
+} from '../../common/fleet/default_setup';
 
-export default function ({ getService }: FtrProviderContext) {
-  const svlCommonApi = getService('svlCommonApi');
-  const supertest = getService('supertest');
+export default function (ctx: FtrProviderContext) {
+  const svlCommonApi = ctx.getService('svlCommonApi');
+  const supertest = ctx.getService('supertest');
 
-  // Failing: See https://github.com/elastic/kibana/issues/176754
-  describe.skip('fleet', function () {
+  describe('fleet', function () {
     it('rejects request to create a new fleet server hosts if host url is different from default', async () => {
+      await expectDefaultFleetServer(ctx);
+
       const { body, status } = await supertest
         .post('/api/fleet/fleet_server_hosts')
         .set(svlCommonApi.getInternalRequestHeader())
@@ -33,6 +38,8 @@ export default function ({ getService }: FtrProviderContext) {
     });
 
     it('accepts request to create a new fleet server hosts if host url is same as default', async () => {
+      await expectDefaultFleetServer(ctx);
+
       const { body, status } = await supertest
         .post('/api/fleet/fleet_server_hosts')
         .set(svlCommonApi.getInternalRequestHeader())
@@ -51,6 +58,8 @@ export default function ({ getService }: FtrProviderContext) {
     });
 
     it('rejects request to create a new elasticsearch output if host is different from default', async () => {
+      await expectDefaultElasticsearchOutput(ctx);
+
       const { body, status } = await supertest
         .post('/api/fleet/outputs')
         .set(svlCommonApi.getInternalRequestHeader())
@@ -71,6 +80,8 @@ export default function ({ getService }: FtrProviderContext) {
     });
 
     it('accepts request to create a new elasticsearch output if host url is same as default', async () => {
+      await expectDefaultElasticsearchOutput(ctx);
+
       const { body, status } = await supertest
         .post('/api/fleet/outputs')
         .set(svlCommonApi.getInternalRequestHeader())