[Fleet] show dataset combo box for input packages (#147015)

## Summary

Closes #145903

Added datasets combo box to input type packages `Dataset name` variable
with the option of creating a new one.

Using the existing `/data_streams` API to show the list of all datasets.

Package policy create/edit API already supports setting the value of
`data_stream.dataset` (input packages should have this variable as
described in #133296)

To verify:
- Start local EPR with input type packages as described
[here](#140035 (comment))
- Add `Custom Logs` integration
- Verify that Dataset name displays a dropdown and the selected value is
persisted.
- Verify that a new value can be entered in Dataset name
- Verify that Edit integration policy displays the existing value and
allows selecting/creating another one.

<img width="924" alt="image"
src="https://user-images.githubusercontent.com/90178898/205680787-3ef7da08-f5f0-4f05-b8d7-3a1c0a6a3d56.png">
<img width="1008" alt="image"
src="https://user-images.githubusercontent.com/90178898/205679497-935fe450-ce78-4f0b-943e-58e7f851f44b.png">
<img width="1006" alt="image"
src="https://user-images.githubusercontent.com/90178898/205679589-fedbbe0e-2c4d-4c00-986f-34ec5c2eb2f6.png">

Added ordering of datasets to move up those that start with the package
name e.g. `system*` datasets come first if adding a `system`
integration. Other than that ordering datasets alphabetically.

<img width="482" alt="image"
src="https://user-images.githubusercontent.com/90178898/205924837-a9807c92-2fe4-431a-88c6-f161d00812fb.png">

The rest of the requirements seem to be already implemented, see
[comments](#145903 (comment))

### Checklist

- [x] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

x-pack/plugins/fleet/cypress/e2e/integrations_real.cy.ts

@@ -174,10 +174,10 @@ describe('Add Integration - Real API', () => {
     setupIntegrations();
     cy.getBySel(getIntegrationCategories('aws')).click();
     cy.getBySel(INTEGRATIONS_SEARCHBAR.BADGE).contains('AWS').should('exist');
-    cy.getBySel(INTEGRATION_LIST).find('.euiCard').should('have.length', 29);
+    cy.getBySel(INTEGRATION_LIST).find('.euiCard').should('have.length.greaterThan', 29);
 
     cy.getBySel(INTEGRATIONS_SEARCHBAR.INPUT).clear().type('Cloud');
-    cy.getBySel(INTEGRATION_LIST).find('.euiCard').should('have.length', 3);
+    cy.getBySel(INTEGRATION_LIST).find('.euiCard').should('have.length.greaterThan', 3);
     cy.getBySel(INTEGRATIONS_SEARCHBAR.REMOVE_BADGE_BUTTON).click();
     cy.getBySel(INTEGRATIONS_SEARCHBAR.BADGE).should('not.exist');
   });

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/dataset_combo.tsx

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React, { useEffect, useState } from 'react';
+import { EuiComboBox } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+
+export const DatasetComboBox: React.FC<{
+  value: any;
+  onChange: (newValue: any) => void;
+  datasets: string[];
+}> = ({ value, onChange, datasets }) => {
+  const datasetOptions = datasets.map((dataset: string) => ({ label: dataset })) ?? [];
+  const defaultOption = 'generic';
+  const [selectedOptions, setSelectedOptions] = useState<Array<{ label: string }>>([
+    {
+      label: value ?? defaultOption,
+    },
+  ]);
+
+  useEffect(() => {
+    if (!value) onChange(defaultOption);
+  }, [value, defaultOption, onChange]);
+
+  const onDatasetChange = (newSelectedOptions: Array<{ label: string }>) => {
+    setSelectedOptions(newSelectedOptions);
+    onChange(newSelectedOptions[0]?.label);
+  };
+
+  const onCreateOption = (searchValue: string = '') => {
+    const normalizedSearchValue = searchValue.trim().toLowerCase();
+    if (!normalizedSearchValue) {
+      return;
+    }
+    const newOption = {
+      label: searchValue,
+    };
+    setSelectedOptions([newOption]);
+    onChange(searchValue);
+  };
+
+  return (
+    <EuiComboBox
+      aria-label={i18n.translate('xpack.fleet.datasetCombo.ariaLabel', {
+        defaultMessage: 'Dataset combo box',
+      })}
+      placeholder={i18n.translate('xpack.fleet.datasetCombo.placeholder', {
+        defaultMessage: 'Select a dataset',
+      })}
+      singleSelection={{ asPlainText: true }}
+      options={datasetOptions}
+      selectedOptions={selectedOptions}
+      onCreateOption={onCreateOption}
+      onChange={onDatasetChange}
+      customOptionText={i18n.translate('xpack.fleet.datasetCombo.customOptionText', {
+        defaultMessage: 'Add {searchValue} as a custom option',
+        values: { searchValue: '{searchValue}' },
+      })}
+      isClearable={false}
+    />
+  );
+};

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/order_datasets.test.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { orderDatasets } from './order_datasets';
+
+describe('orderDatasets', () => {
+  it('should move datasets up that match name', () => {
+    const datasets = orderDatasets(
+      ['system.memory', 'elastic_agent', 'elastic_agent.filebeat', 'system.cpu'],
+      'elastic_agent'
+    );
+
+    expect(datasets).toEqual([
+      'elastic_agent',
+      'elastic_agent.filebeat',
+      'system.cpu',
+      'system.memory',
+    ]);
+  });
+
+  it('should order alphabetically if name does not match', () => {
+    const datasets = orderDatasets(['system.memory', 'elastic_agent'], 'nginx');
+
+    expect(datasets).toEqual(['elastic_agent', 'system.memory']);
+  });
+});

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/order_datasets.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { partition } from 'lodash';
+
+export function orderDatasets(datasetList: string[], name: string): string[] {
+  const [relevantDatasets, otherDatasets] = partition(datasetList.sort(), (record) =>
+    record.startsWith(name)
+  );
+  const datasets = relevantDatasets.concat(otherDatasets);
+  return datasets;
+}

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx

@@ -8,6 +8,7 @@
 import React, { useState, Fragment, memo, useMemo, useEffect, useRef, useCallback } from 'react';
 import ReactMarkdown from 'react-markdown';
 import styled from 'styled-components';
+import { uniq } from 'lodash';
 import { FormattedMessage } from '@kbn/i18n-react';
 import {
   EuiFlexGrid,
@@ -22,6 +23,8 @@ import {
 } from '@elastic/eui';
 import { useRouteMatch } from 'react-router-dom';
 
+import { useGetDataStreams } from '../../../../../../../../hooks';
+
 import { mapPackageReleaseToIntegrationCardRelease } from '../../../../../../../../services/package_prerelease';
 
 import { getRegistryDataStreamAssetBaseName } from '../../../../../../../../../common/services';
@@ -41,6 +44,7 @@ import { PackagePolicyEditorDatastreamMappings } from '../../datastream_mappings
 
 import { PackagePolicyInputVarField } from './package_policy_input_var_field';
 import { useDataStreamId } from './hooks';
+import { orderDatasets } from './order_datasets';
 
 const ScrollAnchor = styled.div`
   display: none;
@@ -175,6 +179,11 @@ export const PackagePolicyInputStreamConfig = memo<Props>(
       });
     };
 
+    const { data: dataStreamsData } = useGetDataStreams();
+    const datasetList =
+      uniq(dataStreamsData?.data_streams.map((dataStream) => dataStream.dataset)) ?? [];
+    const datasets = orderDatasets(datasetList, packageInfo.name);
+
     return (
       <>
         <EuiFlexGrid columns={2}>
@@ -252,6 +261,8 @@ export const PackagePolicyInputStreamConfig = memo<Props>(
                       }}
                       errors={inputStreamValidationResults?.vars![varName]}
                       forceShowErrors={forceShowErrors}
+                      packageType={packageInfo.type}
+                      datasets={datasets}
                     />
                   </EuiFlexItem>
                 );
@@ -311,6 +322,8 @@ export const PackagePolicyInputStreamConfig = memo<Props>(
                             }}
                             errors={inputStreamValidationResults?.vars![varName]}
                             forceShowErrors={forceShowErrors}
+                            packageType={packageInfo.type}
+                            datasets={datasets}
                           />
                         </EuiFlexItem>
                       );