Web Zip file importing

client/dive-common/apispec.ts

@@ -125,7 +125,7 @@ interface Api {
   saveMetadata(datasetId: string, metadata: DatasetMetaMutable): Promise<unknown>;
   saveAttributes(datasetId: string, args: SaveAttributeArgs): Promise<unknown>;
   // Non-Endpoint shared functions
-  openFromDisk(datasetType: DatasetType | 'calibration' | 'annotation' | 'text', directory?: boolean):
+  openFromDisk(datasetType: DatasetType | 'calibration' | 'annotation' | 'text' | 'zip', directory?: boolean):
     Promise<{canceled?: boolean; filePaths: string[]; fileList?: File[]; root?: string}>;
   importAnnotationFile(id: string, path: string, file?: File): Promise<boolean>;
 }

client/dive-common/components/ImportButton.vue

@@ -20,7 +20,7 @@ export default defineComponent({
       required: true,
     },
     openType: {
-      type: String as PropType<DatasetType>,
+      type: String as PropType<DatasetType | 'zip'>,
       required: true,
     },
     multiCamImport: { //TODO: Temporarily used to hide the stereo settings from users
@@ -31,6 +31,10 @@ export default defineComponent({
       type: Object,
       default: () => DefaultButtonAttrs,
     },
+    small: { // Smaller setting for "Add Another ..."
+      type: Boolean,
+      default: false,
+    },
   },
   setup() {
     return {
@@ -50,7 +54,8 @@ export default defineComponent({
       <template v-slot:activator="{ on }">
         <v-btn
           v-bind="buttonAttrs"
-          large
+          :large="!small"
+          :small="small"
           class="px-0"
           @click="$emit('open', openType)"
         >

client/dive-common/constants.ts

@@ -100,6 +100,10 @@ const listFileTypes = [
   'txt',
 ];
 
+const zipFileTypes = [
+  'zip',
+];
+
 const stereoPipelineMarker = 'measurement';
 const multiCamPipelineMarkers = ['2-cam', '3-cam'];
 
@@ -121,6 +125,7 @@ export {
   inputAnnotationTypes,
   inputAnnotationFileTypes,
   listFileTypes,
+  zipFileTypes,
   stereoPipelineMarker,
   multiCamPipelineMarkers,
 };

client/platform/web-girder/utils.ts

@@ -1,6 +1,6 @@
 import {
   calibrationFileTypes, inputAnnotationFileTypes, inputAnnotationTypes,
-  otherImageTypes, otherVideoTypes, websafeImageTypes, websafeVideoTypes,
+  otherImageTypes, otherVideoTypes, websafeImageTypes, websafeVideoTypes, zipFileTypes,
 } from 'dive-common/constants';
 import { DatasetType } from 'dive-common/apispec';
 import type { LocationType, RootlessLocationType } from 'platform/web-girder/store/types';
@@ -34,12 +34,12 @@ function getRouteFromLocation(location: LocationType): string {
   return `/${location._modelType}/${location._id}`;
 }
 
-async function openFromDisk(datasetType: DatasetType | 'calibration' | 'annotation'):
+async function openFromDisk(datasetType: DatasetType | 'calibration' | 'annotation' | 'zip'):
 Promise<{ canceled: boolean; filePaths: string[]; fileList?: File[]}> {
   const input: HTMLInputElement = document.createElement('input');
   input.type = 'file';
   const baseTypes: string[] = inputAnnotationFileTypes.map((item) => `.${item}`);
-  if (!['calbiration', 'annotation'].includes(datasetType)) {
+  if (!['calbiration', 'annotation', 'zip'].includes(datasetType)) {
     input.multiple = true;
   }
   if (datasetType === 'image-sequence') {
@@ -51,6 +51,8 @@ Promise<{ canceled: boolean; filePaths: string[]; fileList?: File[]}> {
   } else if (datasetType === 'annotation') {
     input.accept = inputAnnotationTypes
       .concat(inputAnnotationFileTypes.map((item) => `.${item}`)).join(',');
+  } else if (datasetType === 'zip') {
+    input.accept = zipFileTypes.map((item) => `.${item}`).join(',');
   }
 
   return new Promise(((resolve) => {

client/platform/web-girder/views/Upload.vue

@@ -38,7 +38,7 @@ export interface PendingUpload {
   meta: null | File;
   annotationFile: null | File;
   mediaList: File[];
-  type: DatasetType;
+  type: DatasetType | 'zip';
   fps: number;
   uploading: boolean;
 }
@@ -70,29 +70,63 @@ export default defineComponent({
     const importMultiCamDialog = ref(false);
     const girderUpload: Ref<null | GirderUpload> = ref(null);
     const { prompt } = usePrompt();
-    /**
-     * Initial opening of file dialog
-     */
-    const openImport = async (dstype: DatasetType) => {
-      const ret = await openFromDisk(dstype);
-      if (!ret.canceled && ret.fileList) {
-        // eslint-disable-next-line @typescript-eslint/no-use-before-define
-        const processed = processImport(ret);
-        if (processed?.fullList?.length === 0) return;
-        if (processed && processed.fullList) {
-          const name = processed.fullList.length === 1 ? processed.fullList[0].name : '';
-          preUploadErrorMessage.value = null;
-          try {
-            // eslint-disable-next-line @typescript-eslint/no-use-before-define
-            await addPendingUpload(
-              name, processed.fullList, processed.metaFile,
-              processed.annotationFile, processed.mediaList,
-            );
-          } catch (err) {
-            preUploadErrorMessage.value = err.response?.data?.message || err;
-          }
+
+    const addPendingZipUpload = (name: string, allFiles: File[]) => {
+      const fps = clientSettings.annotationFPS || DefaultVideoFPS;
+      const defaultFilename = allFiles.length ? allFiles[0].name.replace(/\..*/, '') : 'Zip Upload';
+      pendingUploads.value.push({
+        createSubFolders: false,
+        name: defaultFilename,
+        files: [], //Will be set in the GirderUpload Component
+        meta: null,
+        annotationFile: null,
+        mediaList: allFiles,
+        type: 'zip',
+        fps,
+        uploading: false,
+      });
+    };
+
+    const addPendingUpload = async (
+      name: string,
+      allFiles: File[],
+      meta: File | null,
+      annotationFile: File | null,
+      mediaList: File[],
+    ) => {
+      const resp = (await validateUploadGroup(allFiles.map((f) => f.name))).data;
+      if (!resp.ok) {
+        if (resp.message) {
+          preUploadErrorMessage.value = resp.message;
+        }
+        throw new Error(resp.message);
+      }
+      const fps = clientSettings.annotationFPS || DefaultVideoFPS;
+      const defaultFilename = resp.media[0];
+      const validFiles = resp.media.concat(resp.annotations);
+      // mapping needs to be done for the mixin upload functions
+      const internalFiles = allFiles
+        .filter((f) => validFiles.includes(f.name));
+      let createSubFolders = false;
+      if (resp.type === 'video') {
+        if (resp.media.length > 1) {
+          createSubFolders = true;
         }
       }
+      pendingUploads.value.push({
+        createSubFolders,
+        name:
+          internalFiles.length > 1
+            ? defaultFilename.replace(/\..*/, '')
+            : defaultFilename,
+        files: [], //Will be set in the GirderUpload Component
+        meta,
+        annotationFile,
+        mediaList,
+        type: resp.type,
+        fps,
+        uploading: false,
+      });
     };
     /**
      * Processes the imported media files to distinguish between
@@ -157,6 +191,32 @@ export default defineComponent({
       }
       return output;
     };
+    /**
+     * Initial opening of file dialog
+     */
+    const openImport = async (dstype: DatasetType | 'zip') => {
+      const ret = await openFromDisk(dstype);
+      if (!ret.canceled && ret.fileList) {
+        const processed = processImport(ret);
+        if (processed?.fullList?.length === 0) return;
+        if (processed && processed.fullList) {
+          const name = processed.fullList.length === 1 ? processed.fullList[0].name : '';
+          preUploadErrorMessage.value = null;
+          try {
+            if (dstype !== 'zip') {
+              await addPendingUpload(
+                name, processed.fullList, processed.metaFile,
+                processed.annotationFile, processed.mediaList,
+              );
+            } else {
+              addPendingZipUpload(name, processed.fullList);
+            }
+          } catch (err) {
+            preUploadErrorMessage.value = err.response?.data?.message || err;
+          }
+        }
+      }
+    };
     const openMultiCamDialog = (args: { stereo: boolean; openType: 'image-sequence' | 'video' }) => {
       stereo.value = args.stereo;
       multiCamOpenType.value = args.openType;
@@ -204,7 +264,7 @@ export default defineComponent({
           return `${filesNotUploaded(pendingUpload)} files`;
         } if (pendingUpload.type === VideoType && !pendingUpload.uploading) {
           return `${filesNotUploaded(pendingUpload)} videos`;
-        } if (pendingUpload.type === VideoType && pendingUpload.uploading) {
+        } if ((pendingUpload.type === VideoType || pendingUpload.type === 'zip') && pendingUpload.uploading) {
           // For videos we display the total progress when uploading because
           // single videos can be large
           return `${formatSize(totalProgress)} of ${formatSize(totalSize)}`;
@@ -219,52 +279,14 @@ export default defineComponent({
       return `Folder Name${plural}`;
     };
     const getFilenameInputStateDisabled = (pendingUpload: PendingUpload) => (
-      pendingUpload.uploading || pendingUpload.createSubFolders
+      pendingUpload.uploading || (pendingUpload.createSubFolders && pendingUpload.type !== 'zip')
     );
     const getFilenameInputStateHint = (pendingUpload: PendingUpload) => (
-      pendingUpload.createSubFolders ? 'default folder names are used when "Create Subfolders" is selected' : ''
+      (pendingUpload.createSubFolders && pendingUpload.type !== 'zip') ? 'default folder names are used when "Create Subfolders" is selected' : ''
+    );
+    const getFilenameInputValue = (pendingUpload: PendingUpload) => (
+      pendingUpload.createSubFolders && pendingUpload.type !== 'zip' ? 'default' : pendingUpload.name
     );
-    const addPendingUpload = async (
-      name: string,
-      allFiles: File[],
-      meta: File | null,
-      annotationFile: File | null,
-      mediaList: File[],
-    ) => {
-      const resp = (await validateUploadGroup(allFiles.map((f) => f.name))).data;
-      if (!resp.ok) {
-        if (resp.message) {
-          preUploadErrorMessage.value = resp.message;
-        }
-        throw new Error(resp.message);
-      }
-      const fps = clientSettings.annotationFPS || DefaultVideoFPS;
-      const defaultFilename = resp.media[0];
-      const validFiles = resp.media.concat(resp.annotations);
-      // mapping needs to be done for the mixin upload functions
-      const internalFiles = allFiles
-        .filter((f) => validFiles.includes(f.name));
-      let createSubFolders = false;
-      if (resp.type === 'video') {
-        if (resp.media.length > 1) {
-          createSubFolders = true;
-        }
-      }
-      pendingUploads.value.push({
-        createSubFolders,
-        name:
-          internalFiles.length > 1
-            ? defaultFilename.replace(/\..*/, '')
-            : defaultFilename,
-        files: [], //Will be set in the GirderUpload Component
-        meta,
-        annotationFile,
-        mediaList,
-        type: resp.type,
-        fps,
-        uploading: false,
-      });
-    };
     const remove = (pendingUpload: PendingUpload) => {
       const index = pendingUploads.value.indexOf(pendingUpload);
       pendingUploads.value.splice(index, 1);
@@ -323,6 +345,7 @@ export default defineComponent({
       multiCamImport,
       computeUploadProgress,
       getFilenameInputStateLabel,
+      getFilenameInputValue,
       getFilenameInputStateDisabled,
       getFilenameInputStateHint,
       addPendingUpload,
@@ -395,7 +418,7 @@ export default defineComponent({
             <v-row class="align-center">
               <v-col class="py-0">
                 <v-text-field
-                  :value="pendingUpload.createSubFolders ? 'default' : pendingUpload.name"
+                  :value="getFilenameInputValue(pendingUpload)"
                   class="upload-name"
                   :rules="[val => (val || '').length > 0 || 'This field is required']"
                   required
@@ -439,7 +462,7 @@ export default defineComponent({
                 </v-btn>
               </v-col>
             </v-row>
-            <v-row v-if="!pendingUpload.createSubFolders">
+            <v-row v-if="!pendingUpload.createSubFolders && pendingUpload.type !== 'zip'">
               <v-col class="py-0">
                 <v-row>
                   <v-col>
@@ -504,6 +527,7 @@ export default defineComponent({
               icon="mdi-folder-open"
               open-type="image-sequence"
               class="grow"
+              :small="!!pendingUploads.length"
               :class="[pendingUploads.length ? 'mr-3' : 'my-3']"
               :button-attrs="buttonAttrs"
               @open="openImport($event)"
@@ -513,12 +537,23 @@ export default defineComponent({
               :name="`Add ${pendingUploads.length ? 'Another ' : ''}Video`"
               icon="mdi-file-video"
               class="grow"
+              :small="!!pendingUploads.length"
               :class="[pendingUploads.length ? 'ml-3' : 'my-3']"
               open-type="video"
               :button-attrs="buttonAttrs"
               @open="openImport($event)"
               @multi-cam="openMultiCamDialog"
             />
+            <import-button
+              :name="`Add ${pendingUploads.length ? 'Another ' : ''}Zip File`"
+              icon="mdi-zip-box"
+              class="grow"
+              :small="!!pendingUploads.length"
+              :class="[pendingUploads.length ? 'ml-3' : 'my-3']"
+              open-type="zip"
+              :button-attrs="buttonAttrs"
+              @open="openImport($event)"
+            />
           </div>
           <v-btn
             v-if="pendingUploads.length"

docs/Web-Version.md

@@ -34,6 +34,17 @@ A user account is required to store data and run pipelines on viame.kitware.com.
 * Choose a name for the data, enter the optional video playback frame rate, and press start-upload.
 * In the data browser, a new blue "Launch Annotator" button will appear next to your data.
 
+### Zip Files
+
+A zip import can have one of the following file combinations:
+
+* One or more images, an optional annotation file, and an optional configuration file
+* One video with an optional annotation file and an optional configuration file
+* One or more folders which contain the above examples (These will be converted to separate datasets)
+
+Zip import also accepts zip archive files that were generated by DIVE Web's `Download -> Everything` export button.
+
+
 !!! info
 
     All video uploaded to the web server will be transcoded as `mp4/h264`.

server/dive_server/crud.py

@@ -102,7 +102,9 @@ def get_static_pipelines_path() -> Path:
 
 
 def get_or_create_auxiliary_folder(folder, user):
-    return Folder().createFolder(folder, "auxiliary", reuseExisting=True, creator=user)
+    return Folder().createFolder(
+        folder, constants.AuxiliaryFolderName, reuseExisting=True, creator=user
+    )
 
 
 def move_existing_result_to_auxiliary_folder(folder, user):
@@ -111,6 +113,10 @@ def move_existing_result_to_auxiliary_folder(folder, user):
         Item().move(item, auxiliary)
 
 
+def get_or_create_source_folder(folder, user):
+    return Folder().createFolder(folder, "source", reuseExisting=True, creator=user)
+
+
 def itemIsWebsafeVideo(item: Item) -> bool:
     return fromMeta(item, "codec") == "h264"