Add import validation dialog

client/package.json

@@ -47,6 +47,7 @@
     "csv-stringify": "^5.6.0",
     "d3": "^5.12.0",
     "geojs": "1.0.0",
+    "glob-to-regexp": "^0.4.1",
     "lodash": "^4.17.19",
     "moment": "^2.29.1",
     "mousetrap": "^1.6.5",
@@ -65,6 +66,7 @@
     "@types/electron-devtools-installer": "^2.2.0",
     "@types/express": "^4.17.9",
     "@types/geojson": "^7946.0.7",
+    "@types/glob-to-regexp": "^0.4.0",
     "@types/jest": "^25.2.3",
     "@types/lodash": "^4.14.151",
     "@types/mime-types": "^2.1.0",

client/platform/desktop/backend/ipcService.ts

@@ -3,7 +3,7 @@ import http from 'http';
 import { ipcMain } from 'electron';
 
 import {
-  DesktopJobUpdate, RunPipeline, RunTraining, Settings, ExportDatasetArgs,
+  DesktopJobUpdate, RunPipeline, RunTraining, Settings, ExportDatasetArgs, MediaImportPayload,
 } from 'platform/desktop/constants';
 
 import linux from './native/linux';
@@ -59,13 +59,19 @@ export default function register() {
   });
 
   ipcMain.handle('import-media', async (event, { path }: { path: string }) => {
+    const ret = await common.beginMediaImport(
+      settings.get(), path, currentPlatform.checkMedia,
+    );
+    return ret;
+  });
+
+  ipcMain.handle('finalize-import', async (event, args: MediaImportPayload) => {
     const updater = (update: DesktopJobUpdate) => {
       event.sender.send('job-update', update);
     };
-    const ret = await common.importMedia(settings.get(), path, updater, {
-      checkMedia: currentPlatform.checkMedia,
-      convertMedia: currentPlatform.convertMedia,
-    });
+    const ret = await common.finalizeMediaImport(
+      settings.get(), args, updater, currentPlatform.convertMedia,
+    );
     return ret;
   });
 

client/platform/desktop/backend/native/common.spec.ts

@@ -308,44 +308,48 @@ describe('native.common', () => {
   });
 
   it('importMedia image sequence success', async () => {
-    const meta = await common.importMedia(settings, '/home/user/data/imageSuccess', updater, { checkMedia, convertMedia });
-    expect(meta.name).toBe('imageSuccess');
-    expect(meta.originalImageFiles.length).toBe(2);
-    expect(meta.originalVideoFile).toBe('');
-    expect(meta.originalBasePath).toBe('/home/user/data/imageSuccess');
+    const payload = await common.beginMediaImport(settings, '/home/user/data/imageSuccess', checkMedia);
+    expect(payload.jsonMeta.name).toBe('imageSuccess');
+    expect(payload.jsonMeta.originalImageFiles.length).toBe(2);
+    expect(payload.jsonMeta.originalVideoFile).toBe('');
+    expect(payload.jsonMeta.originalBasePath).toBe('/home/user/data/imageSuccess');
   });
 
   it('importMedia video success', async () => {
-    const meta = await common.importMedia(settings, '/home/user/data/videoSuccess/video1.mp4', updater, { checkMedia, convertMedia });
-    expect(meta.name).toBe('video1');
-    expect(meta.originalImageFiles.length).toBe(0);
-    expect(meta.originalVideoFile).toBe('video1.mp4');
-    expect(meta.originalBasePath).toBe('/home/user/data/videoSuccess');
+    const payload = await common.beginMediaImport(settings, '/home/user/data/videoSuccess/video1.mp4', checkMedia);
+    expect(payload.jsonMeta.name).toBe('video1');
+    expect(payload.jsonMeta.originalImageFiles.length).toBe(0);
+    expect(payload.jsonMeta.originalVideoFile).toBe('video1.mp4');
+    expect(payload.jsonMeta.originalBasePath).toBe('/home/user/data/videoSuccess');
   });
 
   it('importMedia empty json file success', async () => {
-    const meta = await common.importMedia(settings, '/home/user/data/annotationEmptySuccess/video1.mp4', updater, { checkMedia, convertMedia });
-    const tracks = await common.loadDetections(settings, meta.id);
+    const payload = await common.beginMediaImport(settings, '/home/user/data/annotationEmptySuccess/video1.mp4', checkMedia);
+    await common.finalizeMediaImport(settings, payload, updater, convertMedia);
+    const tracks = await common.loadDetections(settings, payload.jsonMeta.id);
     expect(tracks).toEqual({});
   });
 
   it('importMedia various failure modes', async () => {
-    await expect(common.importMedia(settings, '/fake/path', updater, { checkMedia, convertMedia }))
+    await expect(common.beginMediaImport(settings, '/fake/path', checkMedia))
       .rejects.toThrow('file or directory not found');
-    await expect(common.importMedia(settings, '/home/user/data/imageSuccess/foo.png', updater, { checkMedia, convertMedia }))
+    await expect(common.beginMediaImport(settings, '/home/user/data/imageSuccess/foo.png', checkMedia))
       .rejects.toThrow('chose image file for video import option');
-    await expect(common.importMedia(settings, '/home/user/data/videoSuccess/otherfile.txt', updater, { checkMedia, convertMedia }))
+    await expect(common.beginMediaImport(settings, '/home/user/data/videoSuccess/otherfile.txt', checkMedia))
       .rejects.toThrow('unsupported MIME type');
-    await expect(common.importMedia(settings, '/home/user/data/videoSuccess/nomime', updater, { checkMedia, convertMedia }))
+    await expect(common.beginMediaImport(settings, '/home/user/data/videoSuccess/nomime', checkMedia))
       .rejects.toThrow('could not determine video MIME');
-    await expect(common.importMedia(settings, '/home/user/data/annotationFail/video1.mp4', updater, { checkMedia, convertMedia }))
+
+    const payload = await common.beginMediaImport(settings, '/home/user/data/annotationFail/video1.mp4', checkMedia);
+    await expect(common.finalizeMediaImport(settings, payload, updater, convertMedia))
       .rejects.toThrow('too many CSV');
   });
 
   it('importMedia video, start conversion', async () => {
-    const meta = await common.importMedia(settings, '/home/user/data/videoSuccess/video1.avi', updater, { checkMedia, convertMedia });
-    expect(meta.transcodingJobKey).toBe('jobKey');
-    expect(meta.type).toBe('video');
+    const payload = await common.beginMediaImport(settings, '/home/user/data/videoSuccess/video1.avi', checkMedia);
+    await common.finalizeMediaImport(settings, payload, updater, convertMedia);
+    expect(payload.jsonMeta.transcodingJobKey).toBe('jobKey');
+    expect(payload.jsonMeta.type).toBe('video');
   });
 
   it('processing good Trained Pipeline folder', async () => {

client/platform/desktop/backend/native/common.ts

@@ -17,11 +17,11 @@ import * as viameSerializers from 'platform/desktop/backend/serializers/viame';
 import {
   websafeImageTypes, websafeVideoTypes, otherImageTypes, otherVideoTypes,
   JsonMeta, Settings, JsonMetaCurrentVersion, DesktopMetadata, DesktopJobUpdater,
-  ConvertMedia, RunTraining, ExportDatasetArgs,
+  ConvertMedia, RunTraining, ExportDatasetArgs, MediaImportPayload,
 } from 'platform/desktop/constants';
+import { cleanString, filterByGlob, makeid } from 'platform/desktop/sharedUtils';
 import { Attribute, Attributes } from 'vue-media-annotator/use/useAttributes';
 import processTrackAttributes from './attributeProcessor';
-import { cleanString, makeid } from './utils';
 
 const ProjectsFolderName = 'DIVE_Projects';
 const JobsFolderName = 'DIVE_Jobs';
@@ -33,6 +33,34 @@ const JsonTrackFileName = /^result(_.*)?\.json$/;
 const JsonMetaFileName = 'meta.json';
 const CsvFileName = /^.*\.csv$/;
 
+async function findImagesInFolder(path: string, glob?: string) {
+  const images: string[] = [];
+  let requiresTranscoding = false;
+  const contents = await fs.readdir(path);
+
+  contents.forEach((filename) => {
+    const abspath = npath.join(path, filename);
+    const mimetype = mime.lookup(abspath);
+    if (glob === undefined || filterByGlob(glob, [filename]).length === 1) {
+      if (
+        mimetype && (websafeImageTypes.includes(mimetype)
+          || otherImageTypes.includes(mimetype))
+      ) {
+        images.push(filename);
+        if (otherImageTypes.includes(mimetype)) {
+          requiresTranscoding = true;
+        }
+      }
+    }
+  });
+  return {
+    images,
+    mediaConvetList: requiresTranscoding
+      ? images.map((filename) => npath.join(path, filename))
+      : [],
+  };
+}
+
 async function _acquireLock(dir: string, resource: string, lockname: 'meta' | 'tracks') {
   const release = await lockfile.lock(resource, {
     stale: 5000, // 5 seconds
@@ -501,21 +529,13 @@ async function _initializeProjectDir(settings: Settings, jsonMeta: JsonMeta): Pr
 }
 
 /**
- * importMedia locates as much information as possible
- * about a dataset using only the directory structure.
- * @param settings user settings
- * @param path path to import dir/file
- * @returns datasetId
+ * Begin a dataset import.
  */
-async function importMedia(
+async function beginMediaImport(
   settings: Settings,
   path: string,
-  updater: DesktopJobUpdater,
-  { checkMedia, convertMedia }: {
-  checkMedia: (settings: Settings, path: string) => Promise<boolean>;
-  convertMedia: ConvertMedia;
-},
-): Promise<JsonMeta> {
+  checkMedia: (settings: Settings, path: string) => Promise<boolean>,
+): Promise<MediaImportPayload> {
   let datasetType: DatasetType;
 
   const exists = fs.existsSync(path);
@@ -556,8 +576,7 @@ async function importMedia(
     jsonMeta.originalBasePath = npath.dirname(path);
   }
 
-  const contents = await fs.readdir(jsonMeta.originalBasePath);
-
+  /* mediaConvertList is a list of absolute paths of media to convert */
   let mediaConvertList: string[] = [];
   /* Extract and validate media from import path */
   if (jsonMeta.type === 'video') {
@@ -578,34 +597,48 @@ async function importMedia(
       throw new Error(`could not determine video MIME type for ${path}`);
     }
   } else if (datasetType === 'image-sequence') {
-    const tempConvertList: string[] = [];
-    let convertAny = false; //If we have to convert one image we convert all for organization
-    contents.forEach((filename) => {
-      const abspath = npath.join(jsonMeta.originalBasePath, filename);
-      const mimetype = mime.lookup(abspath);
-      if (
-        mimetype && (websafeImageTypes.includes(mimetype)
-        || otherImageTypes.includes(mimetype))
-      ) {
-        jsonMeta.originalImageFiles.push(filename);
-        tempConvertList.push(abspath);
-        if (otherImageTypes.includes(mimetype)) {
-          convertAny = true;
-        }
-      }
-    });
-    if (jsonMeta.originalImageFiles.length === 0) {
+    const found = await findImagesInFolder(jsonMeta.originalBasePath);
+    if (found.images.length === 0) {
       throw new Error(`no images found in ${path}`);
     }
-    if (convertAny) {
-      mediaConvertList = tempConvertList;
-    }
+    jsonMeta.originalImageFiles = found.images;
+    mediaConvertList = found.mediaConvetList;
   } else {
     throw new Error('only video and image-sequence types are supported');
   }
 
+  return {
+    jsonMeta,
+    globPattern: '',
+    mediaConvertList,
+  };
+}
+
+/**
+ * Finalize a dataset import.
+ */
+async function finalizeMediaImport(
+  settings: Settings,
+  args: MediaImportPayload,
+  updater: DesktopJobUpdater,
+  convertMedia: ConvertMedia,
+) {
+  const { jsonMeta, globPattern } = args;
+  let { mediaConvertList } = args;
+  const { type: datasetType, id: dsId } = jsonMeta;
+
   const projectDirAbsPath = await _initializeProjectDir(settings, jsonMeta);
 
+  // Filter all parts of the input based on glob pattern
+  if (globPattern && jsonMeta.type === 'image-sequence') {
+    const found = await findImagesInFolder(jsonMeta.originalBasePath, globPattern);
+    if (found.images.length === 0) {
+      throw new Error(`no images in ${jsonMeta.originalBasePath} matched pattern ${globPattern}`);
+    }
+    jsonMeta.originalImageFiles = found.images;
+    mediaConvertList = found.mediaConvetList;
+  }
+
   //Now we will kick off any conversions that are necessary
   let jobBase = null;
   if (mediaConvertList.length) {
@@ -635,7 +668,6 @@ async function importMedia(
     jsonMeta.transcodingJobKey = jobBase.key;
   }
 
-
   let foundDetections = false;
 
   /* Look for JSON track file as first priority */
@@ -654,6 +686,7 @@ async function importMedia(
   }
   /* Look for other types of annotation files as a second priority */
   if (!foundDetections) {
+    const contents = await fs.readdir(jsonMeta.originalBasePath);
     const csvFileCandidates = contents
       .filter((v) => CsvFileName.test(v))
       .map((filename) => npath.join(jsonMeta.originalBasePath, filename));
@@ -712,13 +745,14 @@ async function exportDataset(
 export {
   ProjectsFolderName,
   JobsFolderName,
+  beginMediaImport,
   createKwiverRunWorkingDir,
   exportDataset,
+  finalizeMediaImport,
   getPipelineList,
   getTrainingConfigs,
   getProjectDir,
   getValidatedProjectDir,
-  importMedia,
   loadMetadata,
   loadJsonMetadata,
   loadJsonTracks,

client/platform/desktop/backend/native/utils.ts

@@ -3,24 +3,6 @@ import fs from 'fs-extra';
 import { observeChild } from 'platform/desktop/backend/native/processManager';
 import { DesktopJob, DesktopJobUpdater } from 'platform/desktop/constants';
 
-/**
- * Get a nice safe string
- */
-function cleanString(dirty: string) {
-  return dirty.replace(/[^a-z0-9]/gi, '_').toLowerCase();
-}
-
-// https://stackoverflow.com/questions/1349404/generate-random-string-characters-in-javascript
-function makeid(length: number): string {
-  let result = '';
-  const characters = 'abcdefghijklmnopqrstuvwxyz0123456789';
-  const charactersLength = characters.length;
-  for (let i = 0; i < length; i += 1) {
-    result += characters.charAt(Math.floor(Math.random() * charactersLength));
-  }
-  return result;
-}
-
 const processChunk = (chunk: Buffer) => chunk
   .toString('utf-8')
   .split('\n')
@@ -81,8 +63,6 @@ Promise<{ output: null | string; exitCode: number | null; error: string}> {
 }
 
 export {
-  cleanString,
-  makeid,
   jobFileEchoMiddleware,
   spawnResult,
 };

client/platform/desktop/backend/native/viame.ts

@@ -7,11 +7,12 @@ import {
   ConversionArgs,
   DesktopJobUpdater,
 } from 'platform/desktop/constants';
+import { cleanString } from 'platform/desktop/sharedUtils';
 import { serialize } from 'platform/desktop/backend/serializers/viame';
 import { observeChild } from 'platform/desktop/backend/native/processManager';
 
 import * as common from './common';
-import { cleanString, jobFileEchoMiddleware, spawnResult } from './utils';
+import { jobFileEchoMiddleware, spawnResult } from './utils';
 
 
 const PipelineRelativeDir = 'configs/pipelines';

client/platform/desktop/constants.ts

@@ -186,6 +186,12 @@ export interface DesktopJob {
   endTime?: Date;
 }
 
+export interface MediaImportPayload {
+  jsonMeta: JsonMeta;
+  globPattern: string;
+  mediaConvertList: string[];
+}
+
 export interface DesktopJobUpdate extends DesktopJob {
   // body contents of update payload
   body: string[];

client/platform/desktop/frontend/api.ts

@@ -11,7 +11,7 @@ import type {
 
 import {
   DesktopJob, DesktopMetadata, JsonMeta, NvidiaSmiReply,
-  RunPipeline, RunTraining, fileVideoTypes, ExportDatasetArgs,
+  RunPipeline, RunTraining, fileVideoTypes, ExportDatasetArgs, MediaImportPayload,
 } from 'platform/desktop/constants';
 
 /**
@@ -72,9 +72,12 @@ async function runTraining(
   return ipcRenderer.invoke('run-training', args);
 }
 
-async function importMedia(path: string): Promise<JsonMeta> {
-  const data: JsonMeta = await ipcRenderer.invoke('import-media', { path });
-  return data;
+function importMedia(path: string): Promise<MediaImportPayload> {
+  return ipcRenderer.invoke('import-media', { path });
+}
+
+function finalizeImport(args: MediaImportPayload): Promise<JsonMeta> {
+  return ipcRenderer.invoke('finalize-import', args);
 }
 
 async function exportDataset(id: string, exclude: boolean): Promise<string> {
@@ -143,6 +146,7 @@ export {
   saveAttributes,
   /* Nonstandard APIs */
   exportDataset,
+  finalizeImport,
   importMedia,
   openFromDisk,
   openLink,