fix(import-export): import CSV files with BOM COMPASS-5373

packages/compass-e2e-tests/helpers/selectors.ts

@@ -304,6 +304,7 @@ export const InsertCancel =
 // Import File modal
 
 export const ImportModal = '[data-test-id="import-modal"]';
+export const ImportDelimiter = '[id="import-delimiter-select"]';
 export const ImportFileInput = '#import-file_file_input';
 export const FileTypeJSON = '[data-test-id="select-file-type-json"]';
 export const FileTypeCSV = '[data-test-id="select-file-type-csv"]';

packages/compass-e2e-tests/scripts/insert-data.ts

@@ -61,6 +61,7 @@ if (require.main === module) {
     await createBlankCollection(db, 'json-file');
     await createBlankCollection(db, 'extended-json-file');
     await createBlankCollection(db, 'csv-file');
+    await createBlankCollection(db, 'bom-csv-file');
 
     console.log(`Creating test.numbers`);
     await dropCollection(db, 'numbers');

packages/compass-e2e-tests/tests/collection-import.test.ts

@@ -420,6 +420,91 @@ describe('Collection import', function () {
     });
   });
 
+  it('supports CSV files with BOM', async function () {
+    const csvPath = path.resolve(
+      __dirname,
+      '..',
+      'fixtures',
+      'source-with-bom.csv'
+    );
+
+    await browser.navigateToCollectionTab('test', 'bom-csv-file', 'Documents');
+
+    // open the import modal
+    await browser.clickVisible(Selectors.AddDataButton);
+    const insertDocumentOption = await browser.$(Selectors.ImportFileOption);
+    await insertDocumentOption.waitForDisplayed();
+    await browser.clickVisible(Selectors.ImportFileOption);
+
+    // wait for the modal to appear and select the file
+    const importModal = await browser.$(Selectors.ImportModal);
+    await importModal.waitForDisplayed({ timeout: 10_000 });
+    await browser.selectFile(Selectors.ImportFileInput, csvPath);
+
+    // make sure it auto-selected CSV
+    const fileTypeCSV = await browser.$(Selectors.FileTypeCSV);
+    await browser.waitUntil(async () => {
+      const selected = await fileTypeCSV.getAttribute('aria-selected');
+      return selected === 'true';
+    });
+
+    const selectImportDelimiter = await browser.$(Selectors.ImportDelimiter);
+    await selectImportDelimiter.waitForDisplayed();
+    await selectImportDelimiter.scrollIntoView();
+    await selectImportDelimiter.selectByAttribute('value', ';');
+
+    // pick some types
+    const typeMapping = {
+      amount: 'Number',
+      description: 'String',
+      category: 'Number',
+      name: 'String',
+      order: 'String',
+      color: 'String',
+      date: 'String',
+    };
+
+    for (const [fieldName, fieldType] of Object.entries(typeMapping)) {
+      await selectFieldType(browser, fieldName, fieldType);
+    }
+
+    // confirm
+    await browser.clickVisible(Selectors.ImportConfirm);
+
+    // wait for the done button to appear and then click it
+    const doneButton = await browser.$(Selectors.ImportDone);
+    await doneButton.waitForDisplayed({ timeout: 60_000 });
+
+    await browser.clickVisible(Selectors.ImportDone);
+
+    // wait for the modal to go away
+    await importModal.waitForDisplayed({ reverse: false });
+
+    const messageElement = await browser.$(
+      Selectors.DocumentListActionBarMessage
+    );
+    const text = await messageElement.getText();
+    expect(text).to.equal('Displaying documents 1 - 1 of 1');
+
+    const result = await getFirstListDocument(browser);
+
+    // _id is different every time
+    expect(result._id).to.exist;
+    delete result._id;
+
+    // The values are the text as they appear in the page, so numbers are
+    // strings, strings have double-quotes inside them and the date got
+    // formatted.
+    expect(result).to.deep.equal({
+      amount: '18080',
+      category: '9',
+      name: '"anything"',
+      order: '"9"',
+      date: '"12-01-2016"',
+      // NOTE: amount is a number.
+    });
+  });
+
   it('displays an error if an incompatible type is chosen for a column', async function () {
     const csvPath = path.resolve(__dirname, '..', 'fixtures', 'listings.csv');
 

packages/compass-import-export/src/components/import-options/import-options.jsx

@@ -63,6 +63,7 @@ class ImportOptions extends PureComponent {
               <label className={style('option-select-label')}>
                 Select delimiter
                 <select
+                  id="import-delimiter-select"
                   onChange={(evt) => {
                     this.props.setDelimiter(evt.currentTarget.value);
                   }}

packages/compass-import-export/src/modules/import.js

@@ -397,6 +397,7 @@ const loadPreviewDocs = (
       encoding: 'utf8',
       end: 20 * 1024
     });
+    const stripBOM = stripBomStream();
 
     const dest = createPreviewWritable({
       fileType,
@@ -406,6 +407,7 @@ const loadPreviewDocs = (
 
     stream.pipeline(
       source,
+      stripBOM,
       createPeekStream(fileType, delimiter, fileIsMultilineJSON),
       dest,
       function(err) {

packages/compass-import-export/src/utils/import-preview.spec.js

@@ -4,17 +4,6 @@ import fs from 'fs';
 import path from 'path';
 
 const TEST_DIR = path.join(__dirname, '..', '..', '..', 'test');
-const FIXTURES = {
-  GOOD_CSV: path.join(TEST_DIR, 'good-commas.csv'),
-  BAD_CSV: path.join(TEST_DIR, 'mongoimport', 'test_bad.csv'),
-  JS_I_THINK_IS_JSON: path.join(TEST_DIR, 'js-i-think-is.json'),
-  GOOD_JSON: path.join(TEST_DIR, 'docs.json'),
-  LINE_DELIMITED_JSON: path.join(TEST_DIR, 'docs.jsonl'),
-  LINE_DELIMITED_JSON_EXTRA_LINE: path.join(
-    TEST_DIR,
-    'docs-with-newline-ending.jsonl'
-  )
-};
 
 describe('import-preview', () => {
   describe('createPreviewWritable', () => {
@@ -83,7 +72,7 @@ describe('import-preview', () => {
 
   describe('func', () => {
     it('should return 2 docs for a csv containing 3 docs', (done) => {
-      const src = fs.createReadStream(FIXTURES.GOOD_CSV);
+      const src = fs.createReadStream(path.join(TEST_DIR, 'good-commas.csv'));
       const dest = createPreviewWritable({ MAX_SIZE: 2 });
 
       pipeline(src, createPeekStream('csv'), dest, () => {