[Fleet] Support output and Fleet server proxies on serverless (#175315)

Closes #165672 ### Summary This PR adds support for custom output and Fleet server hosts with proxies in serverless mode: - Proxies are re-enabled in serverless. - It is possible to add custom Fleet server hosts in serverless, with the constraint that the host URL must match the Elasticsearch URL of the default host. - New Elasticsearch outputs must also have the default host URL. ### Testing The below requirements should be tested in serverless mode for observability and security project types: ```sh # elasticsearch yarn es serverless --kill # kibana: one of yarn serverless-oblt yarn serverless-security ``` ⚠️ In addition, stateful mode should **not** be affected by any of these changes. #### Config In order to test this change, you will need the following configuration to mirror a serverless setup. Create a `serverless.dev.yml` if you don't have one already and set a project id (this is required for Fleet's `cloud.isServerlessEnabled` to correctly be `true`): ```yaml xpack.cloud.serverless.project_id: test-123 ``` In `kibana.devl.yml`, make sure the default Fleet server hosts and default output have the expected ids: ```yaml xpack.fleet.fleetServerHosts: - id: default-fleet-server name: Default Fleet server is_default: true host_urls: ['https://host.docker.internal:8220'] xpack.fleet.outputs: - id: es-default-output name: Default output type: elasticsearch is_default: true is_default_monitoring: true hosts: ['https://host.docker.internal:9200'] ``` #### Requirements - [ ] User can create proxy configurations in the Fleet UI and API. - [ ] User can create new Fleet server host via the UI - The Fleet Server Hosts dropdown is disabled and set to the default host URL - [ ] User can create new Fleet server host via the API - The request should succeed if the host URL is set the the default one - Otherwise the request should fail - [ ] User can add a proxy config to the Fleet server host config - [ ] User can select a custom Fleet Server host configuration from the Agent policy settings page - [ ] User can create new Fleet Elasticsearch output via the UI - The Hosts dropdown is disabled and set to the default Elasticsearch URL - [ ] User can create new Fleet Elasticsearch output via the API - The request should succeed if the host URL is set the the default one - Otherwise the request should fail - [ ] User can add a proxy config to the Elasticsearch output - [ ] User can select a custom Elasticsearch output configuration from the Agent policy settings page - [ ] User can create a custom Logstash output with proxy - [ ] User can create a custom Kafka output with proxy ### Checklist - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios ### Screenshots Fleet settings (proxies available): ![Screenshot 2024-02-02 at 17 21 05](https://github.com/elastic/kibana/assets/23701614/e94d3dfa-0467-48d0-9c99-c0288dfbba92) ![Screenshot 2024-02-02 at 17 21 15](https://github.com/elastic/kibana/assets/23701614/eb815d25-0e4c-497f-899a-036a89a0d126) Adding and editing a Fleet server host: ![Screenshot 2024-02-05 at 11 54 04](https://github.com/elastic/kibana/assets/23701614/2ad713fd-7992-4285-8605-1841ab7f56ef) ![Screenshot 2024-02-02 at 17 22 00](https://github.com/elastic/kibana/assets/23701614/dae30411-b023-4a3c-bba2-427dd3b8cd2a) Adding and editing an Elasticsearch output: ![Screenshot 2024-02-02 at 17 22 23](https://github.com/elastic/kibana/assets/23701614/cd6bdb90-d68a-4cf8-a1aa-34cf0eddc978) ![Screenshot 2024-02-02 at 17 22 38](https://github.com/elastic/kibana/assets/23701614/3a9c8f04-2586-4e3a-afd4-7a192245bc15) --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
elastic · Feb 5, 2024 · 11fe56b · 11fe56b
1 parent ff2b79c
commit 11fe56b
Show file tree

Hide file tree

Showing 30 changed files with 693 additions and 184 deletions.
diff --git a/.buildkite/ftr_configs.yml b/.buildkite/ftr_configs.yml
@@ -410,10 +410,12 @@ enabled:
   - x-pack/test_serverless/api_integration/test_suites/observability/config.ts
   - x-pack/test_serverless/api_integration/test_suites/observability/config.feature_flags.ts
   - x-pack/test_serverless/api_integration/test_suites/observability/common_configs/config.group1.ts
+  - x-pack/test_serverless/api_integration/test_suites/observability/fleet/config.ts
   - x-pack/test_serverless/api_integration/test_suites/search/config.ts
   - x-pack/test_serverless/api_integration/test_suites/search/common_configs/config.group1.ts
   - x-pack/test_serverless/api_integration/test_suites/security/config.ts
   - x-pack/test_serverless/api_integration/test_suites/security/common_configs/config.group1.ts
+  - x-pack/test_serverless/api_integration/test_suites/security/fleet/config.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.examples.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.saved_objects_management.ts

diff --git a/config/serverless.yml b/config/serverless.yml
@@ -4,7 +4,6 @@ xpack.serverless.plugin.enabled: true
 # Fleet settings
 xpack.fleet.internal.fleetServerStandalone: true
 xpack.fleet.internal.disableILMPolicies: true
-xpack.fleet.internal.disableProxies: true
 xpack.fleet.internal.activeAgentsSoftLimit: 25000
 xpack.fleet.internal.onlyAllowAgentUpgradeToKnownVersions: true
 xpack.fleet.internal.retrySetupOnBoot: true

diff --git a/test/plugin_functional/test_suites/core_plugins/rendering.ts b/test/plugin_functional/test_suites/core_plugins/rendering.ts
@@ -257,7 +257,6 @@ export default function ({ getService }: PluginFunctionalProviderContext) {
         'xpack.fleet.agents.enabled (boolean)',
         'xpack.fleet.enableExperimental (array)',
         'xpack.fleet.internal.activeAgentsSoftLimit (number)',
-        'xpack.fleet.internal.disableProxies (boolean)',
         'xpack.fleet.internal.fleetServerStandalone (boolean)',
         'xpack.fleet.internal.onlyAllowAgentUpgradeToKnownVersions (boolean)',
         'xpack.fleet.developer.maxAgentPoliciesWithInactivityTimeout (number)',

diff --git a/x-pack/plugins/fleet/README.md b/x-pack/plugins/fleet/README.md
@@ -170,9 +170,11 @@ You can also run a specific test by passing the filepath as an argument, e.g.:
 yarn jest --config x-pack/plugins/fleet/jest.config.js x-pack/plugins/fleet/common/services/validate_package_policy.test.ts
 ```
 
-#### API integration tests
+#### API integration tests (stateful)
 
-You need to have `docker` to run ingest manager api integration tests.
+API integration tests are run using the functional test runner (FTR). When developing or troubleshooting tests, it is convenient to run the server and tests separately as detailed below.
+
+Note: Docker needs to be running to run these tests.
 
 1. In one terminal, run the server from the Kibana root directory with
 
@@ -188,22 +190,38 @@ You need to have `docker` to run ingest manager api integration tests.
 
 1. In a second terminal, run the tests from the Kibana root directory with
 
-   ```
+   ```bash
    FLEET_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:runner --config x-pack/test/fleet_api_integration/<configFile>
    ```
 
    Optionally, you can filter which tests you want to run using `--grep`
 
-   ```
+   ```bash
    FLEET_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:runner --config x-pack/test/fleet_api_integration/<configFile> --grep='fleet'
    ```
 
-**Note** you can also supply which docker image to use for the package registry via the `FLEET_PACKAGE_REGISTRY_DOCKER_IMAGE` env variable. For example,
+Note: you can also supply which Docker image to use for the Package Registry via the `FLEET_PACKAGE_REGISTRY_DOCKER_IMAGE` env variable. For example,
 
-```
+```bash
 FLEET_PACKAGE_REGISTRY_DOCKER_IMAGE='docker.elastic.co/package-registry/distribution:production' FLEET_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:runner
 ```
 
+#### API integration tests (serverless)
+
+The process for running serverless API integration tests is similar as above. Security and observability project types have Fleet enabled. At the time of writing, the same tests exist for Fleet under these two project types.
+
+Security:
+```bash
+FLEET_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:server --config x-pack/test_serverless/api_integration/test_suites/security/fleet/config.ts
+FLEET_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:runner --config  x-pack/test_serverless/api_integration/test_suites/security/fleet/config.ts
+```
+
+Observability:
+```bash
+FLEET_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:server --config x-pack/test_serverless/api_integration/test_suites/observability/fleet/config.ts
+FLEET_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:runner --config  x-pack/test_serverless/api_integration/test_suites/observability/fleet/config.ts
+```
+
 #### Cypress tests
 
 We support UI end-to-end testing with Cypress. Refer to [cypress/README.md](./cypress/README.md) for how to run these tests.

diff --git a/x-pack/plugins/fleet/common/constants/fleet_server_policy_config.ts b/x-pack/plugins/fleet/common/constants/fleet_server_policy_config.ts
@@ -12,3 +12,5 @@ export const DEFAULT_FLEET_SERVER_HOST_ID = 'fleet-default-fleet-server-host';
 export const FLEET_PROXY_SAVED_OBJECT_TYPE = 'fleet-proxy';
 
 export const PROXY_URL_REGEX = /^(http[s]?|socks5):\/\/[^\s$.?#].[^\s]*$/gm;
+
+export const SERVERLESS_DEFAULT_FLEET_SERVER_HOST_ID = 'default-fleet-server';
diff --git a/x-pack/plugins/fleet/common/constants/output.ts b/x-pack/plugins/fleet/common/constants/output.ts
@@ -26,6 +26,8 @@ export const DEFAULT_OUTPUT: NewOutput = {
   hosts: [''],
 };
 
+export const SERVERLESS_DEFAULT_OUTPUT_ID = 'es-default-output';
+
 export const LICENCE_FOR_PER_POLICY_OUTPUT = 'platinum';
 
 /**

diff --git a/x-pack/plugins/fleet/common/types/index.ts b/x-pack/plugins/fleet/common/types/index.ts
@@ -51,7 +51,6 @@ export interface FleetConfigType {
   };
   internal?: {
     disableILMPolicies: boolean;
-    disableProxies: boolean;
     fleetServerStandalone: boolean;
     onlyAllowAgentUpgradeToKnownVersions: boolean;
     activeAgentsSoftLimit?: number;

diff --git a/...c/applications/fleet/sections/agents/agent_list_page/components/search_and_filter_bar.tsx b/...c/applications/fleet/sections/agents/agent_list_page/components/search_and_filter_bar.tsx
@@ -21,7 +21,6 @@ import { useIsFirstTimeAgentUserQuery } from '../../../../../integrations/sectio
 import type { Agent, AgentPolicy } from '../../../../types';
 import { SearchBar } from '../../../../components';
 import { AGENTS_INDEX, AGENTS_PREFIX } from '../../../../constants';
-import { useFleetServerStandalone } from '../../../../hooks';
 
 import { AgentBulkActions } from './bulk_actions';
 import type { SelectionMode } from './types';
@@ -90,10 +89,8 @@ export const SearchAndFilterBar: React.FunctionComponent<SearchAndFilterBarProps
   onClickAgentActivity,
   showAgentActivityTour,
 }) => {
-  const { isFleetServerStandalone } = useFleetServerStandalone();
   const { isFirstTimeAgentUser, isLoading: isFirstTimeAgentUserLoading } =
     useIsFirstTimeAgentUserQuery();
-  const showAddFleetServerBtn = !isFleetServerStandalone;
 
   return (
     <>
@@ -110,25 +107,23 @@ export const SearchAndFilterBar: React.FunctionComponent<SearchAndFilterBarProps
                 showAgentActivityTour={showAgentActivityTour}
               />
             </EuiFlexItem>
-            {showAddFleetServerBtn && (
-              <EuiFlexItem grow={false}>
-                <EuiToolTip
-                  content={
-                    <FormattedMessage
-                      id="xpack.fleet.agentList.addFleetServerButton.tooltip"
-                      defaultMessage="Fleet Server is a component of the Elastic Stack used to centrally manage Elastic Agents"
-                    />
-                  }
-                >
-                  <EuiButton onClick={onClickAddFleetServer} data-test-subj="addFleetServerButton">
-                    <FormattedMessage
-                      id="xpack.fleet.agentList.addFleetServerButton"
-                      defaultMessage="Add Fleet Server"
-                    />
-                  </EuiButton>
-                </EuiToolTip>
-              </EuiFlexItem>
-            )}
+            <EuiFlexItem grow={false}>
+              <EuiToolTip
+                content={
+                  <FormattedMessage
+                    id="xpack.fleet.agentList.addFleetServerButton.tooltip"
+                    defaultMessage="Fleet Server is a component of the Elastic Stack used to centrally manage Elastic Agents"
+                  />
+                }
+              >
+                <EuiButton onClick={onClickAddFleetServer} data-test-subj="addFleetServerButton">
+                  <FormattedMessage
+                    id="xpack.fleet.agentList.addFleetServerButton"
+                    defaultMessage="Add Fleet Server"
+                  />
+                </EuiButton>
+              </EuiToolTip>
+            </EuiFlexItem>
             <EuiFlexItem grow={false}>
               <EuiToolTip
                 content={

diff --git a/...fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx b/...fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx
@@ -65,18 +65,20 @@ import { OutputFormLogstashSection } from './output_form_logstash';
 import { OutputFormElasticsearchSection } from './output_form_elasticsearch';
 
 export interface EditOutputFlyoutProps {
+  defaultOuput?: Output;
   output?: Output;
   onClose: () => void;
   proxies: FleetProxy[];
 }
 
 export const EditOutputFlyout: React.FunctionComponent<EditOutputFlyoutProps> = ({
+  defaultOuput,
   onClose,
   output,
   proxies,
 }) => {
   useBreadcrumbs('settings');
-  const form = useOutputForm(onClose, output);
+  const form = useOutputForm(onClose, output, defaultOuput);
   const inputs = form.inputs;
   const { docLinks, cloud } = useStartServices();
   const { euiTheme } = useEuiTheme();

diff --git a/...tions/fleet/sections/settings/components/edit_output_flyout/output_form_elasticsearch.tsx b/...tions/fleet/sections/settings/components/edit_output_flyout/output_form_elasticsearch.tsx
@@ -36,6 +36,14 @@ export const OutputFormElasticsearchSection: React.FunctionComponent<Props> = (p
         )}
         {...inputs.elasticsearchUrlInput.props}
         isUrl
+        helpText={
+          inputs.elasticsearchUrlInput.props.disabled && (
+            <FormattedMessage
+              id="xpack.fleet.settings.editOutputFlyout.serverlessHostUrlsHelpText"
+              defaultMessage="Custom host URLs are not allowed in serverless."
+            />
+          )
+        }
       />
       <EuiFormRow
         fullWidth

diff --git a/...ic/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.tsx b/...ic/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.tsx
@@ -156,7 +156,7 @@ function extractKafkaOutputSecrets(
   return Object.keys(secrets).length ? secrets : null;
 }
 
-export function useOutputForm(onSucess: () => void, output?: Output) {
+export function useOutputForm(onSucess: () => void, output?: Output, defaultOuput?: Output) {
   const fleetStatus = useFleetStatus();
 
   const { showExperimentalShipperOptions } = ExperimentalFeaturesService.get();
@@ -166,7 +166,7 @@ export function useOutputForm(onSucess: () => void, output?: Output) {
   );
 
   const [isLoading, setIsloading] = useState(false);
-  const { notifications } = useStartServices();
+  const { notifications, cloud } = useStartServices();
   const { confirm } = useConfirmModal();
 
   // preconfigured output do not allow edition
@@ -208,11 +208,17 @@ export function useOutputForm(onSucess: () => void, output?: Output) {
     validateCATrustedFingerPrint,
     isDisabled('ca_trusted_fingerprint')
   );
+  // ES output's host URL is restricted to default in serverless
+  const isServerless = cloud?.isServerlessEnabled;
+  // Set the hosts to default for new ES output in serverless.
+  const elasticsearchUrlDefaultValue =
+    isServerless && !output?.hosts ? defaultOuput?.hosts || [] : output?.hosts || [];
+  const elasticsearchUrlDisabled = isServerless || isDisabled('hosts');
   const elasticsearchUrlInput = useComboInput(
     'esHostsComboxBox',
-    output?.hosts ?? [],
+    elasticsearchUrlDefaultValue,
     validateESHosts,
-    isDisabled('hosts')
+    elasticsearchUrlDisabled
   );
 
   const presetInput = useInput(