[ML] Adds E5 model configs

x-pack/packages/ml/trained_models_utils/index.ts

@@ -19,7 +19,7 @@ export {
   type ModelDefinition,
   type ModelDefinitionResponse,
   type ElserVersion,
-  type GetElserOptions,
+  type GetModelDownloadConfigOptions,
   ELSER_ID_V1,
   ELASTIC_MODEL_TAG,
   ELASTIC_MODEL_TYPE,

x-pack/packages/ml/trained_models_utils/src/constants/trained_models.ts

@@ -89,13 +89,42 @@ export const ELASTIC_MODEL_DEFINITIONS: Record<string, ModelDefinition> = Object
       defaultMessage: 'Elastic Learned Sparse EncodeR v2, optimized for linux-x86_64',
     }),
   },
+  '.multilingual-e5-small': {
+    modelName: 'e5',
+    version: 1,
+    default: true,
+    config: {
+      input: {
+        field_names: ['text_field'],
+      },
+    },
+    description: i18n.translate('xpack.ml.trainedModels.modelsList.e5v1Description', {
+      defaultMessage: 'E5 multilingual embedding model',
+    }),
+  },
+  '.multilingual-e5-small_linux-x86_64': {
+    modelName: 'e5',
+    version: 1,
+    os: 'Linux',
+    arch: 'amd64',
+    config: {
+      input: {
+        field_names: ['text_field'],
+      },
+    },
+    description: i18n.translate('xpack.ml.trainedModels.modelsList.e5v1Description', {
+      defaultMessage: 'E5 multilingual embedding model, optimized for linux-x86_64',
+    }),
+  },
 } as const);
 
+export type ElasticCurateModelName = 'elser' | 'e5';
+
 export interface ModelDefinition {
   /**
    * Model name, e.g. elser
    */
-  modelName: string;
+  modelName: ElasticCurateModelName;
   version: number;
   /**
    * Default PUT model configuration
@@ -129,6 +158,6 @@ export type ModelState = typeof MODEL_STATE[keyof typeof MODEL_STATE] | null;
 
 export type ElserVersion = 1 | 2;
 
-export interface GetElserOptions {
+export interface GetModelDownloadConfigOptions {
   version?: ElserVersion;
 }

x-pack/plugins/ml/public/application/model_management/add_model_flyout.tsx

@@ -187,6 +187,38 @@ const ElserTabContent: FC<ElserTabContentProps> = ({ modelDownloads, onModelDown
               </div>
             ) : null}
 
+            {modelName === 'e5' ? (
+              <div>
+                <EuiTitle size={'s'}>
+                  <h3>
+                    <FormattedMessage
+                      id="xpack.ml.trainedModels.modelsList.e5Title"
+                      defaultMessage="E5 multilingual embedding model"
+                    />
+                  </h3>
+                </EuiTitle>
+                <EuiSpacer size="s" />
+                <p>
+                  <EuiText color={'subdued'} size={'s'}>
+                    <FormattedMessage
+                      id="xpack.ml.trainedModels.addModelFlyout.e5Description"
+                      defaultMessage="E5 produces dense vectors embeddings that can be searched in multiple languages."
+                    />
+                  </EuiText>
+                </p>
+                <EuiSpacer size="s" />
+                <p>
+                  <EuiLink href={docLinks.links.ml.nlpElser} external>
+                    <FormattedMessage
+                      id="xpack.ml.trainedModels.modelsList.elserViewDocumentationLinkLabel"
+                      defaultMessage="View documentation"
+                    />
+                  </EuiLink>
+                </p>
+                <EuiSpacer size={'m'} />
+              </div>
+            ) : null}
+
             <EuiFormFieldset
               legend={{
                 children: (

x-pack/plugins/ml/public/application/services/elastic_models_service.ts

@@ -5,7 +5,10 @@
  * 2.0.
  */
 
-import type { ModelDefinitionResponse, GetElserOptions } from '@kbn/ml-trained-models-utils';
+import type {
+  ModelDefinitionResponse,
+  GetModelDownloadConfigOptions,
+} from '@kbn/ml-trained-models-utils';
 import { type TrainedModelsApiService } from './ml_api_service/trained_models';
 
 export class ElasticModels {
@@ -17,7 +20,7 @@ export class ElasticModels {
    * If any of the ML nodes run a different OS rather than Linux, or the CPU architecture isn't x86_64,
    * a portable version of the model is returned.
    */
-  public async getELSER(options?: GetElserOptions): Promise<ModelDefinitionResponse> {
+  public async getELSER(options?: GetModelDownloadConfigOptions): Promise<ModelDefinitionResponse> {
     return await this.trainedModels.getElserConfig(options);
   }
 }

x-pack/plugins/ml/public/application/services/ml_api_service/trained_models.ts

@@ -11,7 +11,10 @@ import type { IngestPipeline } from '@elastic/elasticsearch/lib/api/types';
 import { useMemo } from 'react';
 import type { HttpFetchQuery } from '@kbn/core/public';
 import type { ErrorType } from '@kbn/ml-error-utils';
-import type { GetElserOptions, ModelDefinitionResponse } from '@kbn/ml-trained-models-utils';
+import type {
+  GetModelDownloadConfigOptions,
+  ModelDefinitionResponse,
+} from '@kbn/ml-trained-models-utils';
 import { ML_INTERNAL_BASE_PATH } from '../../../../common/constants/app';
 import type { MlSavedObjectType } from '../../../../common/types/saved_objects';
 import { HttpService } from '../http_service';
@@ -73,7 +76,7 @@ export function trainedModelsApiProvider(httpService: HttpService) {
     /**
      * Gets ELSER config for download based on the cluster OS and CPU architecture.
      */
-    getElserConfig(options?: GetElserOptions) {
+    getElserConfig(options?: GetModelDownloadConfigOptions) {
       return httpService.http<ModelDefinitionResponse>({
         path: `${ML_INTERNAL_BASE_PATH}/trained_models/elser_config`,
         method: 'GET',

x-pack/plugins/ml/server/models/model_management/model_provider.test.ts

@@ -76,6 +76,24 @@ describe('modelsProvider', () => {
           version: 2,
           modelName: 'elser',
         },
+        {
+          config: { input: { field_names: ['text_field'] } },
+          description: 'E5 multilingual embedding model',
+          name: '.multilingual-e5-small',
+          default: true,
+          version: 1,
+          modelName: 'e5',
+        },
+        {
+          arch: 'amd64',
+          config: { input: { field_names: ['text_field'] } },
+          description: 'E5 multilingual embedding model, optimized for linux-x86_64',
+          name: '.multilingual-e5-small_linux-x86_64',
+          os: 'Linux',
+          recommended: true,
+          version: 1,
+          modelName: 'e5',
+        },
       ]);
     });
 
@@ -129,6 +147,23 @@ describe('modelsProvider', () => {
           version: 2,
           modelName: 'elser',
         },
+        {
+          config: { input: { field_names: ['text_field'] } },
+          description: 'E5 multilingual embedding model',
+          name: '.multilingual-e5-small',
+          recommended: true,
+          version: 1,
+          modelName: 'e5',
+        },
+        {
+          arch: 'amd64',
+          config: { input: { field_names: ['text_field'] } },
+          description: 'E5 multilingual embedding model, optimized for linux-x86_64',
+          name: '.multilingual-e5-small_linux-x86_64',
+          os: 'Linux',
+          version: 1,
+          modelName: 'e5',
+        },
       ]);
     });
   });
@@ -175,4 +210,37 @@ describe('modelsProvider', () => {
       expect(result.name).toEqual('.elser_model_2_linux-x86_64');
     });
   });
+
+  describe('getCuratedModelConfig', () => {
+    test('provides a recommended definition by default', async () => {
+      const result = await modelService.getCuratedModelConfig('e5');
+      expect(result.name).toEqual('.multilingual-e5-small_linux-x86_64');
+    });
+
+    test('provides a default version if there is no recommended', async () => {
+      mockCloud.cloudId = undefined;
+      (mockClient.asInternalUser.transport.request as jest.Mock).mockResolvedValueOnce({
+        _nodes: {
+          total: 1,
+          successful: 1,
+          failed: 0,
+        },
+        cluster_name: 'default',
+        nodes: {
+          yYmqBqjpQG2rXsmMSPb9pQ: {
+            name: 'node-0',
+            roles: ['ml'],
+            attributes: {},
+            os: {
+              name: 'Mac OS X',
+              arch: 'aarch64',
+            },
+          },
+        },
+      });
+
+      const result = await modelService.getCuratedModelConfig('e5');
+      expect(result.name).toEqual('.multilingual-e5-small');
+    });
+  });
 });

x-pack/plugins/ml/server/models/model_management/models_provider.ts

@@ -19,10 +19,11 @@ import type {
 } from '@elastic/elasticsearch/lib/api/types';
 import {
   ELASTIC_MODEL_DEFINITIONS,
-  type GetElserOptions,
+  type GetModelDownloadConfigOptions,
   type ModelDefinitionResponse,
 } from '@kbn/ml-trained-models-utils';
 import type { CloudSetup } from '@kbn/cloud-plugin/server';
+import type { ElasticCurateModelName } from '@kbn/ml-trained-models-utils/src/constants/trained_models';
 import type { PipelineDefinition } from '../../../common/types/trained_models';
 import type { MlClient } from '../../lib/ml_client';
 import type { MLSavedObjectService } from '../../saved_objects';
@@ -52,6 +53,8 @@ interface ModelMapResult {
   error: null | any;
 }
 
+export type GetCuratedModelConfigParams = Parameters<ModelsProvider['getCuratedModelConfig']>;
+
 export class ModelsProvider {
   private _transforms?: TransformGetTransformTransformSummary[];
 
@@ -410,8 +413,6 @@ export class ModelsProvider {
       }
       throw error;
     }
-
-    return result;
   }
 
   /**
@@ -494,14 +495,19 @@ export class ModelsProvider {
   }
 
   /**
-   * Provides an ELSER model name and configuration for download based on the current cluster architecture.
-   * The current default version is 2. If running on Cloud it returns the Linux x86_64 optimized version.
-   * If any of the ML nodes run a different OS rather than Linux, or the CPU architecture isn't x86_64,
-   * a portable version of the model is returned.
+   * Provides an appropriate model ID and configuration for download based on the current cluster architecture.
+   *
+   * @param modelName
+   * @param options
+   * @returns
    */
-  async getELSER(options?: GetElserOptions): Promise<ModelDefinitionResponse> | never {
-    const modelDownloadConfig = await this.getModelDownloads();
-
+  async getCuratedModelConfig(
+    modelName: ElasticCurateModelName,
+    options?: GetModelDownloadConfigOptions
+  ): Promise<ModelDefinitionResponse> | never {
+    const modelDownloadConfig = (await this.getModelDownloads()).filter(
+      (model) => model.modelName === modelName
+    );
     let requestedModel: ModelDefinitionResponse | undefined;
     let recommendedModel: ModelDefinitionResponse | undefined;
     let defaultModel: ModelDefinitionResponse | undefined;
@@ -527,6 +533,18 @@ export class ModelsProvider {
     return requestedModel || recommendedModel || defaultModel!;
   }
 
+  /**
+   * Provides an ELSER model name and configuration for download based on the current cluster architecture.
+   * The current default version is 2. If running on Cloud it returns the Linux x86_64 optimized version.
+   * If any of the ML nodes run a different OS rather than Linux, or the CPU architecture isn't x86_64,
+   * a portable version of the model is returned.
+   */
+  async getELSER(
+    options?: GetModelDownloadConfigOptions
+  ): Promise<ModelDefinitionResponse> | never {
+    return await this.getCuratedModelConfig('elser', options);
+  }
+
   /**
    * Puts the requested ELSER model into elasticsearch, triggering elasticsearch to download the model.
    * Assigns the model to the * space.

x-pack/plugins/ml/server/shared_services/providers/trained_models.ts

@@ -8,14 +8,18 @@
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import type { CloudSetup } from '@kbn/cloud-plugin/server';
 import type { KibanaRequest, SavedObjectsClientContract } from '@kbn/core/server';
-import type { GetElserOptions, ModelDefinitionResponse } from '@kbn/ml-trained-models-utils';
+import type {
+  GetModelDownloadConfigOptions,
+  ModelDefinitionResponse,
+} from '@kbn/ml-trained-models-utils';
 import type {
   MlInferTrainedModelRequest,
   MlStopTrainedModelDeploymentRequest,
   UpdateTrainedModelDeploymentRequest,
   UpdateTrainedModelDeploymentResponse,
 } from '../../lib/ml_client/types';
 import { modelsProvider } from '../../models/model_management';
+import type { GetCuratedModelConfigParams } from '../../models/model_management/models_provider';
 import type { GetGuards } from '../shared_services';
 
 export interface TrainedModelsProvider {
@@ -47,7 +51,7 @@ export interface TrainedModelsProvider {
     putTrainedModel(
       params: estypes.MlPutTrainedModelRequest
     ): Promise<estypes.MlPutTrainedModelResponse>;
-    getELSER(params?: GetElserOptions): Promise<ModelDefinitionResponse>;
+    getELSER(params?: GetModelDownloadConfigOptions): Promise<ModelDefinitionResponse>;
   };
 }
 
@@ -123,14 +127,22 @@ export function getTrainedModelsProvider(
               return mlClient.putTrainedModel(params);
             });
         },
-        async getELSER(params?: GetElserOptions) {
+        async getELSER(params?: GetModelDownloadConfigOptions) {
           return await guards
             .isFullLicense()
             .hasMlCapabilities(['canGetTrainedModels'])
             .ok(async ({ scopedClient, mlClient }) => {
               return modelsProvider(scopedClient, mlClient, cloud).getELSER(params);
             });
         },
+        async getCuratedModelConfig(...params: GetCuratedModelConfigParams) {
+          return await guards
+            .isFullLicense()
+            .hasMlCapabilities(['canGetTrainedModels'])
+            .ok(async ({ scopedClient, mlClient }) => {
+              return modelsProvider(scopedClient, mlClient, cloud).getCuratedModelConfig(...params);
+            });
+        },
       };
     },
   };