Merge branch 'main' into 178491-bucket-exception

.buildkite/scripts/steps/serverless/build_and_deploy.sh

@@ -154,7 +154,7 @@ $BUILDKITE_PULL_REQUEST
 
 ### Further details
 
-Caused by @$GITHUB_PR_TRIGGER_USER using the github label in https://github.com/elastic/kibana/pull/$BUILDKITE_PULL_REQUEST
+Caused by the GitHub label 'ci:project-deploy-observability' in https://github.com/elastic/kibana/pull/$BUILDKITE_PULL_REQUEST
 EOF
 
   GH_TOKEN="$GITHUB_TOKEN" \

packages/kbn-search-connectors/types/native_connectors.ts

@@ -2808,84 +2808,89 @@ export const NATIVE_CONNECTOR_DEFINITIONS: Record<string, NativeConnector | unde
   },
   notion: {
     configuration: {
-      tenant_id: {
+      notion_secret_key: {
         default_value: null,
         depends_on: [],
         display: DisplayType.TEXTBOX,
-        label: i18n.translate('searchConnectors.nativeConnectors.notion.tenantIdLabel', {
-          defaultMessage: 'Tenant ID',
+        label: i18n.translate('searchConnectors.nativeConnectors.notion.notionSecretKeyLabel', {
+          defaultMessage: 'Notion Secret Key',
         }),
         options: [],
         order: 1,
         required: true,
-        sensitive: false,
+        sensitive: true,
         tooltip: null,
         type: FieldType.STRING,
         ui_restrictions: [],
         validations: [],
         value: '',
       },
-      client_id: {
-        default_value: null,
+      databases: {
+        default_value: '',
         depends_on: [],
         display: DisplayType.TEXTBOX,
-        label: i18n.translate('searchConnectors.nativeConnectors.notion.clientIdLabel', {
-          defaultMessage: 'Client ID',
+        label: i18n.translate('searchConnectors.nativeConnectors.notion.databasesLabel', {
+          defaultMessage: 'List of Databases',
         }),
         options: [],
         order: 2,
         required: true,
         sensitive: false,
-        tooltip: null,
-        type: FieldType.STRING,
+        tooltip: '',
+        type: FieldType.LIST,
         ui_restrictions: [],
         validations: [],
         value: '',
       },
-      secret_value: {
-        default_value: null,
+      pages: {
+        default_value: '',
         depends_on: [],
         display: DisplayType.TEXTBOX,
-        label: i18n.translate('searchConnectors.nativeConnectors.notion.secretValueLabel', {
-          defaultMessage: 'Secret value',
+        label: i18n.translate('searchConnectors.nativeConnectors.notion.pagesLabel', {
+          defaultMessage: 'List of Pages',
         }),
         options: [],
         order: 3,
         required: true,
-        sensitive: true,
-        tooltip: null,
-        type: FieldType.STRING,
+        sensitive: false,
+        tooltip: '',
+        type: FieldType.LIST,
         ui_restrictions: [],
         validations: [],
         value: '',
       },
-      username: {
+      index_comments: {
         default_value: null,
         depends_on: [],
-        display: DisplayType.TEXTBOX,
-        label: USERNAME_LABEL,
+        display: DisplayType.TOGGLE,
+        label: i18n.translate('searchConnectors.nativeConnectors.notion.indexCommentsLabel', {
+          defaultMessage: 'Enable indexing comments',
+        }),
         options: [],
         order: 4,
         required: true,
         sensitive: false,
-        tooltip: null,
-        type: FieldType.STRING,
+        tooltip: i18n.translate('searchConnectors.nativeConnectors.notion.indexCommentsTooltip', {
+          defaultMessage:
+            'Enabling this will increase the amount of network calls to the source, and may decrease performance',
+        }),
+        type: FieldType.BOOLEAN,
         ui_restrictions: [],
         validations: [],
-        value: '',
+        value: false,
       },
-      password: {
-        default_value: null,
+      concurrent_downloads: {
+        default_value: 30,
         depends_on: [],
-        display: DisplayType.TEXTBOX,
-        label: PASSWORD_LABEL,
+        display: DisplayType.NUMERIC,
+        label: MAX_CONCURRENT_DOWNLOADS_LABEL,
         options: [],
         order: 5,
-        required: true,
-        sensitive: true,
+        required: false,
+        sensitive: false,
         tooltip: null,
-        type: FieldType.STRING,
-        ui_restrictions: [],
+        type: FieldType.INTEGER,
+        ui_restrictions: ['advanced'],
         validations: [],
         value: '',
       },

src/dev/build/tasks/os_packages/docker_generator/templates/base/Dockerfile

@@ -25,21 +25,27 @@ RUN cd /tmp && \
      https://{{publicArtifactSubdomain}}.elastic.co/downloads/kibana/{{artifactPrefix}}-$(arch).tar.gz && \
   cd -
 {{/usePublicArtifact}}
-
 {{^usePublicArtifact}}
 COPY {{artifactTarball}} /tmp/kibana.tar.gz
 {{/usePublicArtifact}}
 
 RUN mkdir /usr/share/kibana
 WORKDIR /usr/share/kibana
-RUN tar --strip-components=1 -zxf /tmp/kibana.tar.gz
+RUN tar \
+# Exclude serverless.yml disabled assets
+{{#serverless}}
+  --exclude=screenshotting-plugin/chromium \
+  --exclude=screenshotting-plugin/server/assets \
+{{/serverless}}
+  --strip-components=1 \
+  -zxf /tmp/kibana.tar.gz
 # Ensure that group permissions are the same as user permissions.
 # This will help when relying on GID-0 to run Kibana, rather than UID-1000.
 # OpenShift does this, for example.
 # REF: https://docs.openshift.org/latest/creating_images/guidelines.html
 RUN chmod -R g=u /usr/share/kibana
-
 {{#cloud}}
+
 COPY {{filebeatTarball}} /tmp/filebeat.tar.gz
 COPY {{metricbeatTarball}} /tmp/metricbeat.tar.gz
 

x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/create_field/semantic_text/use_semantic_text.test.ts

@@ -13,10 +13,15 @@ import { act } from 'react-dom/test-utils';
 const mlMock: any = {
   mlApi: {
     inferenceModels: {
-      createInferenceEndpoint: jest.fn(),
+      createInferenceEndpoint: jest.fn().mockResolvedValue({}),
     },
     trainedModels: {
-      startModelAllocation: jest.fn(),
+      startModelAllocation: jest.fn().mockResolvedValue({}),
+      getTrainedModels: jest.fn().mockResolvedValue([
+        {
+          fully_defined: true,
+        },
+      ]),
     },
   },
 };
@@ -93,6 +98,11 @@ describe('useSemanticText', () => {
       result.current.handleSemanticText(mockFieldData);
     });
 
+    expect(mlMock.mlApi.trainedModels.startModelAllocation).toHaveBeenCalledWith('.elser_model_2');
+    expect(mockDispatch).toHaveBeenCalledWith({
+      type: 'field.addSemanticText',
+      value: mockFieldData,
+    });
     expect(mlMock.mlApi.inferenceModels.createInferenceEndpoint).toHaveBeenCalledWith(
       'elser_model_2',
       'text_embedding',
@@ -105,16 +115,58 @@ describe('useSemanticText', () => {
         },
       }
     );
-    expect(mlMock.mlApi.trainedModels.startModelAllocation).toHaveBeenCalledWith('.elser_model_2');
-    expect(mockDispatch).toHaveBeenCalledWith({
-      type: 'field.addSemanticText',
-      value: mockFieldData,
+  });
+
+  it('should invoke the download api if the model does not exist', async () => {
+    const mlMockWithModelNotDownloaded: any = {
+      mlApi: {
+        inferenceModels: {
+          createInferenceEndpoint: jest.fn(),
+        },
+        trainedModels: {
+          startModelAllocation: jest.fn(),
+          getTrainedModels: jest.fn().mockResolvedValue([
+            {
+              fully_defined: false,
+            },
+          ]),
+          installElasticTrainedModelConfig: jest.fn().mockResolvedValue({}),
+        },
+      },
+    };
+    const { result } = renderHook(() =>
+      useSemanticText({
+        form,
+        setErrorsInTrainedModelDeployment: jest.fn(),
+        ml: mlMockWithModelNotDownloaded,
+      })
+    );
+
+    await act(async () => {
+      result.current.handleSemanticText(mockFieldData);
+    });
+
+    expect(
+      mlMockWithModelNotDownloaded.mlApi.trainedModels.installElasticTrainedModelConfig
+    ).toHaveBeenCalledWith('.elser_model_2');
+    expect(
+      mlMockWithModelNotDownloaded.mlApi.trainedModels.startModelAllocation
+    ).toHaveBeenCalledWith('.elser_model_2');
+    expect(
+      mlMockWithModelNotDownloaded.mlApi.inferenceModels.createInferenceEndpoint
+    ).toHaveBeenCalledWith('elser_model_2', 'text_embedding', {
+      service: 'elasticsearch',
+      service_settings: {
+        num_allocations: 1,
+        num_threads: 1,
+        model_id: '.elser_model_2',
+      },
     });
   });
 
   it('handles errors correctly', async () => {
     const mockError = new Error('Test error');
-    mlMock.mlApi.inferenceModels.createInferenceEndpoint.mockImplementationOnce(() => {
+    mlMock.mlApi?.trainedModels.startModelAllocation.mockImplementationOnce(() => {
       throw mockError;
     });
 

x-pack/plugins/index_management/public/application/components/mappings_editor/components/document_fields/fields/create_field/semantic_text/use_semantic_text.ts

@@ -6,7 +6,8 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import { MlPluginStart } from '@kbn/ml-plugin/public';
+import { useCallback } from 'react';
+import { MlPluginStart, TrainedModelConfigResponse } from '@kbn/ml-plugin/public';
 import React, { useEffect, useState } from 'react';
 import { useComponentTemplatesContext } from '../../../../../../component_templates/component_templates_context';
 import { useDispatch, useMappingsState } from '../../../../../mappings_state_context';
@@ -64,20 +65,37 @@ export function useSemanticText(props: UseSemanticTextProps) {
     }
   }, [form, inferenceId, inferenceToModelIdMap]);
 
-  const handleSemanticText = (data: Field) => {
-    data.inferenceId = inferenceValue;
+  const isModelDownloaded = useCallback(
+    async (modelId: string) => {
+      try {
+        const response: TrainedModelConfigResponse[] | undefined =
+          await ml?.mlApi?.trainedModels.getTrainedModels(modelId, {
+            include: 'definition_status',
+          });
+        return !!response?.[0]?.fully_defined;
+      } catch (error) {
+        if (error.body.statusCode !== 404) {
+          throw error;
+        }
+      }
+      return false;
+    },
+    [ml?.mlApi?.trainedModels]
+  );
+
+  const createInferenceEndpoint = (
+    trainedModelId: string,
+    defaultInferenceEndpoint: boolean,
+    data: Field
+  ) => {
     if (data.inferenceId === undefined) {
-      return;
-    }
-
-    const inferenceData = inferenceToModelIdMap?.[data.inferenceId];
-
-    if (!inferenceData) {
-      return;
+      throw new Error(
+        i18n.translate('xpack.idxMgmt.mappingsEditor.createField.undefinedInferenceIdError', {
+          defaultMessage: 'InferenceId is undefined while creating the inference endpoint.',
+        })
+      );
     }
 
-    const { trainedModelId, defaultInferenceEndpoint, isDeployed, isDeployable } = inferenceData;
-
     if (trainedModelId && defaultInferenceEndpoint) {
       const modelConfig = {
         service: 'elasticsearch',
@@ -87,28 +105,45 @@ export function useSemanticText(props: UseSemanticTextProps) {
           model_id: trainedModelId,
         },
       };
-      try {
-        ml?.mlApi?.inferenceModels?.createInferenceEndpoint(
-          data.inferenceId,
-          'text_embedding',
-          modelConfig
-        );
-      } catch (error) {
-        setErrorsInTrainedModelDeployment?.((prevItems) => [...prevItems, trainedModelId]);
-        toasts?.addError(error.body && error.body.message ? new Error(error.body.message) : error, {
-          title: i18n.translate(
-            'xpack.idxMgmt.mappingsEditor.createField.inferenceEndpointCreationErrorTitle',
-            {
-              defaultMessage: 'Inference endpoint creation failed',
-            }
-          ),
-        });
-      }
+
+      ml?.mlApi?.inferenceModels?.createInferenceEndpoint(
+        data.inferenceId,
+        'text_embedding',
+        modelConfig
+      );
     }
+  };
+
+  const handleSemanticText = async (data: Field) => {
+    data.inferenceId = inferenceValue;
+    if (data.inferenceId === undefined) {
+      return;
+    }
+
+    const inferenceData = inferenceToModelIdMap?.[data.inferenceId];
+
+    if (!inferenceData) {
+      return;
+    }
+
+    const { trainedModelId, defaultInferenceEndpoint, isDeployed, isDeployable } = inferenceData;
 
-    if (isDeployable && trainedModelId && !isDeployed) {
+    if (isDeployable && trainedModelId) {
       try {
-        ml?.mlApi?.trainedModels.startModelAllocation(trainedModelId);
+        const modelDownloaded: boolean = await isModelDownloaded(trainedModelId);
+
+        if (isDeployed) {
+          createInferenceEndpoint(trainedModelId, defaultInferenceEndpoint, data);
+        } else if (modelDownloaded) {
+          ml?.mlApi?.trainedModels
+            .startModelAllocation(trainedModelId)
+            .then(() => createInferenceEndpoint(trainedModelId, defaultInferenceEndpoint, data));
+        } else {
+          ml?.mlApi?.trainedModels
+            .installElasticTrainedModelConfig(trainedModelId)
+            .then(() => ml?.mlApi?.trainedModels.startModelAllocation(trainedModelId))
+            .then(() => createInferenceEndpoint(trainedModelId, defaultInferenceEndpoint, data));
+        }
         toasts?.addSuccess({
           title: i18n.translate(
             'xpack.idxMgmt.mappingsEditor.createField.modelDeploymentStartedNotification',

x-pack/plugins/ml/public/index.ts

@@ -26,6 +26,7 @@ export const plugin: PluginInitializer<
 > = (initializerContext: PluginInitializerContext) => new MlPlugin(initializerContext);
 
 export type { MlPluginSetup, MlPluginStart };
+export type { TrainedModelConfigResponse } from '../common/types/trained_models';
 
 export type { MlCapabilitiesResponse } from '../common/types/capabilities';
 export type { MlSummaryJob } from '../common/types/anomaly_detection_jobs';

x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/workflows/basic_license_essentials_tier/exception_comments_serverless.ts

@@ -26,7 +26,8 @@ export default ({ getService }: FtrProviderContext) => {
   const log = getService('log');
   const supertestWithoutAuth = getService('supertestWithoutAuth');
 
-  describe('@serverless exception item comments - serverless specific behavior', () => {
+  // Skipping in MKI due to roles testing not yet being available
+  describe('@serverless @skipInServerlessMKI exception item comments - serverless specific behavior', () => {
     // FLAKY: https://github.com/elastic/kibana/issues/181507
     describe.skip('Rule Exceptions', () => {
       afterEach(async () => {