[file_upload] move ml Importer classes to file_upload plugin (#91559)

* [ml] move importer to file_upload plugin * move file_parse logic into GeoJsonImporter * move file_parser tests to geojson_importer tests * rename geo_json_clean_and_validate to geojson_clean_and_validate * replace file_upload import with Importer.import * simplify JsonIndexFilePicker props * tslint * i18n fixes and tslint fixes * update functional test to account for change in layer name * review feedback * dependency_cache review feedback Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
elastic · Feb 22, 2021 · 6ef7d2c · 6ef7d2c
1 parent b9de3c7
commit 6ef7d2c
Show file tree

Hide file tree

Showing 45 changed files with 901 additions and 1,206 deletions.
diff --git a/x-pack/plugins/file_upload/common/constants.ts b/x-pack/plugins/file_upload/common/constants.ts
@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+export const MB = Math.pow(2, 20);
 export const MAX_FILE_SIZE = '100MB';
 export const MAX_FILE_SIZE_BYTES = 104857600; // 100MB
 

diff --git a/x-pack/plugins/file_upload/common/types.ts b/x-pack/plugins/file_upload/common/types.ts
@@ -26,7 +26,7 @@ export interface Doc {
   message: string;
 }
 
-export type ImportDoc = Doc | string;
+export type ImportDoc = Doc | string | object;
 
 export interface Settings {
   pipeline?: string;

diff --git a/x-pack/plugins/file_upload/public/api/index.ts b/x-pack/plugins/file_upload/public/api/index.ts
@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { FileUploadComponentProps, lazyLoadFileUploadModules } from '../lazy_load_bundle';
+import type { IImporter, ImportFactoryOptions } from '../importer';
+
+export interface FileUploadStartApi {
+  getFileUploadComponent(): Promise<React.ComponentType<FileUploadComponentProps>>;
+  importerFactory(format: string, options: ImportFactoryOptions): Promise<IImporter | undefined>;
+}
+
+export async function getFileUploadComponent(): Promise<
+  React.ComponentType<FileUploadComponentProps>
+> {
+  const fileUploadModules = await lazyLoadFileUploadModules();
+  return fileUploadModules.JsonUploadAndParse;
+}
+
+export async function importerFactory(
+  format: string,
+  options: ImportFactoryOptions
+): Promise<IImporter | undefined> {
+  const fileUploadModules = await lazyLoadFileUploadModules();
+  return fileUploadModules.importerFactory(format, options);
+}
diff --git a/x-pack/plugins/file_upload/public/components/json_import_progress.js b/x-pack/plugins/file_upload/public/components/json_import_progress.js
@@ -9,7 +9,7 @@ import React, { Fragment, Component } from 'react';
 import { i18n } from '@kbn/i18n';
 import { EuiCodeBlock, EuiSpacer, EuiText, EuiTitle, EuiProgress, EuiCallOut } from '@elastic/eui';
 import { FormattedMessage } from '@kbn/i18n/react';
-import { basePath } from '../kibana_services';
+import { getHttp } from '../kibana_services';
 
 export class JsonImportProgress extends Component {
   state = {
@@ -118,7 +118,7 @@ export class JsonImportProgress extends Component {
                 <a
                   data-test-subj="indexManagementNewIndexLink"
                   target="_blank"
-                  href={`${basePath}/app/management/kibana/indexPatterns`}
+                  href={getHttp().basePath.prepend('/app/management/kibana/indexPatterns')}
                 >
                   {i18n.translate('xpack.fileUpload.jsonImport.indexMgmtLink', {
                     defaultMessage: 'Index Management',

diff --git a/x-pack/plugins/file_upload/public/components/json_index_file_picker.js b/x-pack/plugins/file_upload/public/components/json_index_file_picker.js
@@ -9,7 +9,6 @@ import React, { Fragment, Component } from 'react';
 import { EuiFilePicker, EuiFormRow, EuiProgress } from '@elastic/eui';
 import { FormattedMessage } from '@kbn/i18n/react';
 import { i18n } from '@kbn/i18n';
-import { parseFile } from '../util/file_parser';
 
 const MAX_FILE_SIZE = 52428800;
 const ACCEPTABLE_FILETYPES = ['json', 'geojson'];
@@ -33,7 +32,7 @@ export class JsonIndexFilePicker extends Component {
     this._isMounted = false;
   }
 
-  getFileParseActive = () => this._isMounted && this.state.fileParseActive;
+  isFileParseActive = () => this._isMounted && this.state.fileParseActive;
 
   _fileHandler = (fileList) => {
     const fileArr = Array.from(fileList);
@@ -61,36 +60,6 @@ export class JsonIndexFilePicker extends Component {
     );
   };
 
-  _checkFileSize = ({ size }) => {
-    const fileSizeValid = true;
-    try {
-      if (size > MAX_FILE_SIZE) {
-        const humanReadableSize = bytesToSize(size);
-        const humanReadableMaxSize = bytesToSize(MAX_FILE_SIZE);
-        throw new Error(
-          i18n.translate('xpack.fileUpload.jsonIndexFilePicker.acceptableFileSize', {
-            defaultMessage: 'File size {fileSize} exceeds max file size of {maxFileSize}',
-            values: {
-              fileSize: humanReadableSize,
-              maxFileSize: humanReadableMaxSize,
-            },
-          })
-        );
-      }
-    } catch (error) {
-      this.setState({
-        fileUploadError: i18n.translate('xpack.fileUpload.jsonIndexFilePicker.fileSizeError', {
-          defaultMessage: 'File size error: {errorMessage}',
-          values: {
-            errorMessage: error.message,
-          },
-        }),
-      });
-      return;
-    }
-    return fileSizeValid;
-  };
-
   _getFileNameAndCheckType({ name }) {
     let fileNameOnly;
     try {
@@ -136,54 +105,58 @@ export class JsonIndexFilePicker extends Component {
 
   setFileProgress = ({ featuresProcessed, bytesProcessed, totalBytes }) => {
     const percentageProcessed = parseInt((100 * bytesProcessed) / totalBytes);
-    if (this.getFileParseActive()) {
+    if (this.isFileParseActive()) {
       this.setState({ featuresProcessed, percentageProcessed });
     }
   };
 
   async _parseFile(file) {
     const { currentFileTracker } = this.state;
-    const {
-      setFileRef,
-      setParsedFile,
-      resetFileAndIndexSettings,
-      onFileUpload,
-      transformDetails,
-      setIndexName,
-    } = this.props;
+    const { setFileRef, setParsedFile, resetFileAndIndexSettings } = this.props;
+
+    if (file.size > MAX_FILE_SIZE) {
+      this.setState({
+        fileUploadError: i18n.translate('xpack.fileUpload.jsonIndexFilePicker.acceptableFileSize', {
+          defaultMessage: 'File size {fileSize} exceeds maximum file size of {maxFileSize}',
+          values: {
+            fileSize: bytesToSize(file.size),
+            maxFileSize: bytesToSize(MAX_FILE_SIZE),
+          },
+        }),
+      });
+      resetFileAndIndexSettings();
+      return;
+    }
 
-    const fileSizeValid = this._checkFileSize(file);
     const defaultIndexName = this._getFileNameAndCheckType(file);
-    if (!fileSizeValid || !defaultIndexName) {
+    if (!defaultIndexName) {
       resetFileAndIndexSettings();
       return;
     }
-    // Parse file
 
-    const fileResult = await parseFile({
-      file,
-      transformDetails,
-      onFileUpload,
-      setFileProgress: this.setFileProgress,
-      getFileParseActive: this.getFileParseActive,
-    }).catch((err) => {
-      if (this._isMounted) {
-        this.setState({
-          fileParseActive: false,
-          percentageProcessed: 0,
-          featuresProcessed: 0,
-          fileUploadError: (
-            <FormattedMessage
-              id="xpack.fileUpload.jsonIndexFilePicker.unableParseFile"
-              defaultMessage="Unable to parse file: {error}"
-              values={{
-                error: err.message,
-              }}
-            />
-          ),
-        });
-      }
-    });
+    const fileResult = await this.props.geojsonImporter
+      .readFile(file, this.setFileProgress, this.isFileParseActive)
+      .catch((err) => {
+        if (this._isMounted) {
+          this.setState({
+            fileParseActive: false,
+            percentageProcessed: 0,
+            featuresProcessed: 0,
+            fileUploadError: (
+              <FormattedMessage
+                id="xpack.fileUpload.jsonIndexFilePicker.unableParseFile"
+                defaultMessage="Unable to parse file: {error}"
+                values={{
+                  error: err.message,
+                }}
+              />
+            ),
+          });
+          resetFileAndIndexSettings();
+          return;
+        }
+      });
+
     if (!this._isMounted) {
       return;
     }
@@ -198,25 +171,20 @@ export class JsonIndexFilePicker extends Component {
       resetFileAndIndexSettings();
       return;
     }
-    const { errors, parsedGeojson } = fileResult;
 
-    if (errors.length) {
-      // Set only the first error for now (since there's only one).
-      // TODO: Add handling in case of further errors
-      const error = errors[0];
+    if (fileResult.errors.length) {
       this.setState({
         fileUploadError: (
           <FormattedMessage
             id="xpack.fileUpload.jsonIndexFilePicker.fileParseError"
             defaultMessage="File parse error(s) detected: {error}"
-            values={{ error }}
+            values={{ error: fileResult.errors[0] }}
           />
         ),
       });
     }
-    setIndexName(defaultIndexName);
     setFileRef(file);
-    setParsedFile(parsedGeojson);
+    setParsedFile(fileResult, defaultIndexName);
   }
 
   render() {