From f9aaecf5a7882bc0e52f108975648ce7008102a2 Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Thu, 18 Nov 2021 10:15:22 -0500
Subject: [PATCH 01/16] initial MVP of zip upload working

---
 client/dive-common/apispec.ts                 |  2 +-
 .../dive-common/components/ImportButton.vue   |  9 ++-
 client/dive-common/constants.ts               |  5 ++
 client/platform/web-girder/utils.ts           |  8 ++-
 client/platform/web-girder/views/Upload.vue   | 58 +++++++++++++++----
 server/dive_server/crud_rpc.py                | 25 ++++++++
 server/dive_tasks/tasks.py                    | 43 ++++++++++++++
 server/dive_utils/constants.py                |  2 +
 8 files changed, 135 insertions(+), 17 deletions(-)

diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts
index 1c7dee2f4..9d751ff10 100644
--- a/client/dive-common/apispec.ts
+++ b/client/dive-common/apispec.ts
@@ -125,7 +125,7 @@ interface Api {
   saveMetadata(datasetId: string, metadata: DatasetMetaMutable): Promise<unknown>;
   saveAttributes(datasetId: string, args: SaveAttributeArgs): Promise<unknown>;
   // Non-Endpoint shared functions
-  openFromDisk(datasetType: DatasetType | 'calibration' | 'annotation' | 'text', directory?: boolean):
+  openFromDisk(datasetType: DatasetType | 'calibration' | 'annotation' | 'text' | 'zip', directory?: boolean):
     Promise<{canceled?: boolean; filePaths: string[]; fileList?: File[]; root?: string}>;
   importAnnotationFile(id: string, path: string, file?: File): Promise<boolean>;
 }
diff --git a/client/dive-common/components/ImportButton.vue b/client/dive-common/components/ImportButton.vue
index e203d1382..577b870b9 100644
--- a/client/dive-common/components/ImportButton.vue
+++ b/client/dive-common/components/ImportButton.vue
@@ -20,7 +20,7 @@ export default defineComponent({
       required: true,
     },
     openType: {
-      type: String as PropType<DatasetType>,
+      type: String as PropType<DatasetType | 'zip'>,
       required: true,
     },
     multiCamImport: { //TODO: Temporarily used to hide the stereo settings from users
@@ -31,6 +31,10 @@ export default defineComponent({
       type: Object,
       default: () => DefaultButtonAttrs,
     },
+    small: { // Smaller setting for "Add Another ..."
+      type: Boolean,
+      default: false,
+    },
   },
   setup() {
     return {
@@ -50,7 +54,8 @@ export default defineComponent({
       <template v-slot:activator="{ on }">
         <v-btn
           v-bind="buttonAttrs"
-          large
+          :large="!small"
+          :small="small"
           class="px-0"
           @click="$emit('open', openType)"
         >
diff --git a/client/dive-common/constants.ts b/client/dive-common/constants.ts
index cb3017232..192d2c4c5 100644
--- a/client/dive-common/constants.ts
+++ b/client/dive-common/constants.ts
@@ -100,6 +100,10 @@ const listFileTypes = [
   'txt',
 ];
 
+const zipFileTypes = [
+  'zip',
+];
+
 const stereoPipelineMarker = 'measurement';
 const multiCamPipelineMarkers = ['2-cam', '3-cam'];
 
@@ -121,6 +125,7 @@ export {
   inputAnnotationTypes,
   inputAnnotationFileTypes,
   listFileTypes,
+  zipFileTypes,
   stereoPipelineMarker,
   multiCamPipelineMarkers,
 };
diff --git a/client/platform/web-girder/utils.ts b/client/platform/web-girder/utils.ts
index 8daac9086..a2675670e 100644
--- a/client/platform/web-girder/utils.ts
+++ b/client/platform/web-girder/utils.ts
@@ -1,6 +1,6 @@
 import {
   calibrationFileTypes, inputAnnotationFileTypes, inputAnnotationTypes,
-  otherImageTypes, otherVideoTypes, websafeImageTypes, websafeVideoTypes,
+  otherImageTypes, otherVideoTypes, websafeImageTypes, websafeVideoTypes, zipFileTypes,
 } from 'dive-common/constants';
 import { DatasetType } from 'dive-common/apispec';
 import type { LocationType, RootlessLocationType } from 'platform/web-girder/store/types';
@@ -34,12 +34,12 @@ function getRouteFromLocation(location: LocationType): string {
   return `/${location._modelType}/${location._id}`;
 }
 
-async function openFromDisk(datasetType: DatasetType | 'calibration' | 'annotation'):
+async function openFromDisk(datasetType: DatasetType | 'calibration' | 'annotation' | 'zip'):
 Promise<{ canceled: boolean; filePaths: string[]; fileList?: File[]}> {
   const input: HTMLInputElement = document.createElement('input');
   input.type = 'file';
   const baseTypes: string[] = inputAnnotationFileTypes.map((item) => `.${item}`);
-  if (!['calbiration', 'annotation'].includes(datasetType)) {
+  if (!['calbiration', 'annotation', 'zip'].includes(datasetType)) {
     input.multiple = true;
   }
   if (datasetType === 'image-sequence') {
@@ -51,6 +51,8 @@ Promise<{ canceled: boolean; filePaths: string[]; fileList?: File[]}> {
   } else if (datasetType === 'annotation') {
     input.accept = inputAnnotationTypes
       .concat(inputAnnotationFileTypes.map((item) => `.${item}`)).join(',');
+  } else if (datasetType === 'zip') {
+    input.accept = zipFileTypes.map((item) => `.${item}`).join(',');
   }
 
   return new Promise(((resolve) => {
diff --git a/client/platform/web-girder/views/Upload.vue b/client/platform/web-girder/views/Upload.vue
index efcc101af..2d5383106 100644
--- a/client/platform/web-girder/views/Upload.vue
+++ b/client/platform/web-girder/views/Upload.vue
@@ -38,7 +38,7 @@ export interface PendingUpload {
   meta: null | File;
   annotationFile: null | File;
   mediaList: File[];
-  type: DatasetType;
+  type: DatasetType | 'zip';
   fps: number;
   uploading: boolean;
 }
@@ -73,7 +73,7 @@ export default defineComponent({
     /**
      * Initial opening of file dialog
      */
-    const openImport = async (dstype: DatasetType) => {
+    const openImport = async (dstype: DatasetType | 'zip') => {
       const ret = await openFromDisk(dstype);
       if (!ret.canceled && ret.fileList) {
         // eslint-disable-next-line @typescript-eslint/no-use-before-define
@@ -83,11 +83,16 @@ export default defineComponent({
           const name = processed.fullList.length === 1 ? processed.fullList[0].name : '';
           preUploadErrorMessage.value = null;
           try {
+            if (dstype !== 'zip') {
             // eslint-disable-next-line @typescript-eslint/no-use-before-define
-            await addPendingUpload(
-              name, processed.fullList, processed.metaFile,
-              processed.annotationFile, processed.mediaList,
-            );
+              await addPendingUpload(
+                name, processed.fullList, processed.metaFile,
+                processed.annotationFile, processed.mediaList,
+              );
+            } else {
+              // eslint-disable-next-line @typescript-eslint/no-use-before-define
+              addPendingZipUpload(name, processed.fullList);
+            }
           } catch (err) {
             preUploadErrorMessage.value = err.response?.data?.message || err;
           }
@@ -204,7 +209,7 @@ export default defineComponent({
           return `${filesNotUploaded(pendingUpload)} files`;
         } if (pendingUpload.type === VideoType && !pendingUpload.uploading) {
           return `${filesNotUploaded(pendingUpload)} videos`;
-        } if (pendingUpload.type === VideoType && pendingUpload.uploading) {
+        } if ((pendingUpload.type === VideoType || pendingUpload.type === 'zip') && pendingUpload.uploading) {
           // For videos we display the total progress when uploading because
           // single videos can be large
           return `${formatSize(totalProgress)} of ${formatSize(totalSize)}`;
@@ -219,11 +224,29 @@ export default defineComponent({
       return `Folder Name${plural}`;
     };
     const getFilenameInputStateDisabled = (pendingUpload: PendingUpload) => (
-      pendingUpload.uploading || pendingUpload.createSubFolders
+      pendingUpload.uploading || (pendingUpload.createSubFolders && pendingUpload.type !== 'zip')
     );
     const getFilenameInputStateHint = (pendingUpload: PendingUpload) => (
-      pendingUpload.createSubFolders ? 'default folder names are used when "Create Subfolders" is selected' : ''
+      (pendingUpload.createSubFolders && pendingUpload.type !== 'zip') ? 'default folder names are used when "Create Subfolders" is selected' : ''
     );
+    const getFilenameInputValue = (pendingUpload: PendingUpload) => (
+      pendingUpload.createSubFolders && pendingUpload.type !== 'zip' ? 'default' : pendingUpload.name
+    );
+    const addPendingZipUpload = (name: string, allFiles: File[]) => {
+      const fps = clientSettings.annotationFPS || DefaultVideoFPS;
+      const defaultFilename = allFiles.length ? allFiles[0].name.replace(/\..*/, '') : 'Zip Upload';
+      pendingUploads.value.push({
+        createSubFolders: false,
+        name: defaultFilename,
+        files: [], //Will be set in the GirderUpload Component
+        meta: null,
+        annotationFile: null,
+        mediaList: allFiles,
+        type: 'zip',
+        fps,
+        uploading: false,
+      });
+    };
     const addPendingUpload = async (
       name: string,
       allFiles: File[],
@@ -323,6 +346,7 @@ export default defineComponent({
       multiCamImport,
       computeUploadProgress,
       getFilenameInputStateLabel,
+      getFilenameInputValue,
       getFilenameInputStateDisabled,
       getFilenameInputStateHint,
       addPendingUpload,
@@ -395,7 +419,7 @@ export default defineComponent({
             <v-row class="align-center">
               <v-col class="py-0">
                 <v-text-field
-                  :value="pendingUpload.createSubFolders ? 'default' : pendingUpload.name"
+                  :value="getFilenameInputValue(pendingUpload)"
                   class="upload-name"
                   :rules="[val => (val || '').length > 0 || 'This field is required']"
                   required
@@ -439,7 +463,7 @@ export default defineComponent({
                 </v-btn>
               </v-col>
             </v-row>
-            <v-row v-if="!pendingUpload.createSubFolders">
+            <v-row v-if="!pendingUpload.createSubFolders && !pendingUpload.type == 'zip'">
               <v-col class="py-0">
                 <v-row>
                   <v-col>
@@ -504,6 +528,7 @@ export default defineComponent({
               icon="mdi-folder-open"
               open-type="image-sequence"
               class="grow"
+              :small="!!pendingUploads.length"
               :class="[pendingUploads.length ? 'mr-3' : 'my-3']"
               :button-attrs="buttonAttrs"
               @open="openImport($event)"
@@ -513,12 +538,23 @@ export default defineComponent({
               :name="`Add ${pendingUploads.length ? 'Another ' : ''}Video`"
               icon="mdi-file-video"
               class="grow"
+              :small="!!pendingUploads.length"
               :class="[pendingUploads.length ? 'ml-3' : 'my-3']"
               open-type="video"
               :button-attrs="buttonAttrs"
               @open="openImport($event)"
               @multi-cam="openMultiCamDialog"
             />
+            <import-button
+              :name="`Add ${pendingUploads.length ? 'Another ' : ''}Zip File`"
+              icon="mdi-zip-box"
+              class="grow"
+              :small="!!pendingUploads.length"
+              :class="[pendingUploads.length ? 'ml-3' : 'my-3']"
+              open-type="zip"
+              :button-attrs="buttonAttrs"
+              @open="openImport($event)"
+            />
           </div>
           <v-btn
             v-if="pendingUploads.length"
diff --git a/server/dive_server/crud_rpc.py b/server/dive_server/crud_rpc.py
index 8a5a41088..8b6fcdf10 100644
--- a/server/dive_server/crud_rpc.py
+++ b/server/dive_server/crud_rpc.py
@@ -368,6 +368,7 @@ def postprocess(
         Transcoding of Video
         Transcoding of Images
         Conversion of KPF annotations into track JSON
+        Extraction and upload of zip files
 
     In either case, the following may run synchronously:
         Conversion of CSV annotations into track JSON
@@ -386,6 +387,30 @@ def postprocess(
 
     if not skipJobs and not isClone:
         token = Token().createToken(user=user, days=2)
+
+        # extract ZIP Files if not already completed
+        zipItems = Folder().childItems(
+            dsFolder,
+            filters={
+                "lowerName": {"$regex": constants.zipRegex},
+                f"meta.{constants.ZipFileExtractedMarker}": {'$exists': False},
+            },
+        )
+        for item in zipItems:
+            newjob = tasks.extract_zip.apply_async(
+                queue=_get_queue_name(user),
+                kwargs=dict(
+                    folderId=str(item["folderId"]),
+                    itemId=str(item["_id"]),
+                    girder_job_title=f"Extracting {item['_id']} to folder {str(dsFolder['_id'])}",
+                    girder_client_token=str(token["_id"]),
+                    girder_job_type="private" if job_is_private else "convert",
+                ),
+            )
+            newjob.job[constants.JOBCONST_PRIVATE_QUEUE] = job_is_private
+            Job().save(newjob.job)
+            return
+
         # transcode VIDEO if necessary
         videoItems = Folder().childItems(
             dsFolder, filters={"lowerName": {"$regex": constants.videoRegex}}
diff --git a/server/dive_tasks/tasks.py b/server/dive_tasks/tasks.py
index 0603ed450..639e035bd 100644
--- a/server/dive_tasks/tasks.py
+++ b/server/dive_tasks/tasks.py
@@ -556,3 +556,46 @@ def convert_images(self: Task, folderId):
             str(folderId),
             {"annotate": True},  # mark the parent folder as able to annotate.
         )
+
+
+@app.task(bind=True, acks_late=True, ignore_result=True)
+def extract_zip(self: Task, folderId: str, itemId: str):
+    context: dict = {}
+    gc: GirderClient = self.girder_client
+    manager: JobManager = patch_manager(self.job_manager)
+    if utils.check_canceled(self, context):
+        manager.updateStatus(JobStatus.CANCELED)
+        return
+
+    with tempfile.TemporaryDirectory() as _working_directory, suppress(utils.CanceledError):
+        _working_directory_path = Path(_working_directory)
+        item: GirderModel = gc.getItem(itemId)
+        file_name = str(_working_directory_path / item['name'])
+        manager.write(f'Fetching input from {itemId} to {file_name}...\n')
+        gc.downloadItem(itemId, _working_directory, item["name"])
+        with zipfile.ZipFile(file_name, 'r') as zipObj:
+            listOfFileNames = zipObj.namelist()
+            for fileName in listOfFileNames:
+                manager.write(f"Extracting: {fileName}\n")
+                zipObj.extract(fileName, f'{_working_directory}/{fileName}')
+        # validation of files in folder using dive/data
+        validation = gc.sendRestRequest(
+            'POST', '/dive_dataset/validate_files', json=listOfFileNames
+        )
+        print(validation)
+        if validation:
+            manager.write(f"Annotations: {validation['annotations']}\n")
+            manager.write(f"Media: {validation['media']}\n")
+            dataset_type = validation['type']
+            manager.write(f"Type: {dataset_type}\n")
+            if "ok" in validation and validation["ok"] is False:
+                manager.write(f"Message: {validation['message']}\n")
+                manager.updateStatus(JobStatus.ERROR)
+                return
+            # Upload all resulting items back into the root folder
+            manager.updateStatus(JobStatus.PUSHING_OUTPUT)
+            gc.upload(f'{_working_directory}/**/*', folderId)
+            gc.addMetadataToFolder(str(folderId), {constants.TypeMarker: dataset_type})
+            gc.addMetadataToItem(str(itemId), {constants.ZipFileExtractedMarker: True})
+            # After uploading the default files we do a the postprocess for video conversion now
+            gc.sendRestRequest("POST", f"/dive_rpc/postprocess/{str(folderId)}")
diff --git a/server/dive_utils/constants.py b/server/dive_utils/constants.py
index 5b3b5c391..ef0518e3a 100644
--- a/server/dive_utils/constants.py
+++ b/server/dive_utils/constants.py
@@ -28,6 +28,7 @@
 csvRegex = re.compile(r"\.csv$", re.IGNORECASE)
 jsonRegex = re.compile(r"\.json$", re.IGNORECASE)
 ymlRegex = re.compile(r"\.ya?ml$", re.IGNORECASE)
+zipRegex = re.compile(r"\.zip$", re.IGNORECASE)
 
 ImageMimeTypes = {
     "image/png",
@@ -75,6 +76,7 @@
 OriginalFPSMarker = "originalFps"
 OriginalFPSStringMarker = "originalFpsString"
 ConfidenceFiltersMarker = "confidenceFilters"
+ZipFileExtractedMarker = "extractedZip"
 
 # Other constants
 TrainedPipelineCategory = "trained"

From d62ce6af9d0a3b1cb02325edba709edb88697115 Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Thu, 18 Nov 2021 10:37:14 -0500
Subject: [PATCH 02/16] mend

---
 server/dive_server/crud_rpc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/dive_server/crud_rpc.py b/server/dive_server/crud_rpc.py
index 8b6fcdf10..1274965a8 100644
--- a/server/dive_server/crud_rpc.py
+++ b/server/dive_server/crud_rpc.py
@@ -409,7 +409,7 @@ def postprocess(
             )
             newjob.job[constants.JOBCONST_PRIVATE_QUEUE] = job_is_private
             Job().save(newjob.job)
-            return
+            return dsFolder
 
         # transcode VIDEO if necessary
         videoItems = Folder().childItems(

From de2f662d7b4447c18c5d0d6a99a419e678f1aa4f Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Fri, 19 Nov 2021 09:52:42 -0500
Subject: [PATCH 03/16] adding in some failure conditions

---
 server/dive_tasks/tasks.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/server/dive_tasks/tasks.py b/server/dive_tasks/tasks.py
index 639e035bd..0af348204 100644
--- a/server/dive_tasks/tasks.py
+++ b/server/dive_tasks/tasks.py
@@ -575,7 +575,22 @@ def extract_zip(self: Task, folderId: str, itemId: str):
         gc.downloadItem(itemId, _working_directory, item["name"])
         with zipfile.ZipFile(file_name, 'r') as zipObj:
             listOfFileNames = zipObj.namelist()
+            sum_file_size = sum([data.file_size for data in zipObj.filelist])
+            sum_compress_size = sum([data.compress_size for data in zipObj.filelist])
+            ratio = sum_file_size / sum_compress_size
+            if ratio > 600:
+                manager.write(
+                    f"Compression ratio is exceedingly high at {ratio}\n\
+                    Please contact an admin at viame-web@kitware.com if this is a valid zip file"
+                )
+                manager.updateStatus(JobStatus.ERROR)
+                return
+
             for fileName in listOfFileNames:
+                if fileName.endswith('.zip'):
+                    manager.write("Nested Zip files are invalid!!!!\n")
+                    manager.updateStatus(JobStatus.ERROR)
+                    return
                 manager.write(f"Extracting: {fileName}\n")
                 zipObj.extract(fileName, f'{_working_directory}/{fileName}')
         # validation of files in folder using dive/data
@@ -591,7 +606,6 @@ def extract_zip(self: Task, folderId: str, itemId: str):
             if "ok" in validation and validation["ok"] is False:
                 manager.write(f"Message: {validation['message']}\n")
                 manager.updateStatus(JobStatus.ERROR)
-                return
             # Upload all resulting items back into the root folder
             manager.updateStatus(JobStatus.PUSHING_OUTPUT)
             gc.upload(f'{_working_directory}/**/*', folderId)

From 0a5385b3ce667e21240903df88f86c4b60de3ed4 Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Mon, 22 Nov 2021 16:07:16 -0500
Subject: [PATCH 04/16] adding in integration testing

---
 README.md                                    |  2 +
 docs/Web-Version.md                          |  5 ++
 server/tests/integration/conftest.py         | 26 ++++++++
 server/tests/integration/test_zip_uploads.py | 66 ++++++++++++++++++++
 4 files changed, 99 insertions(+)
 create mode 100644 server/tests/integration/test_zip_uploads.py

diff --git a/README.md b/README.md
index 4423e2394..26f4551f2 100644
--- a/README.md
+++ b/README.md
@@ -37,6 +37,8 @@ DIVE takes two different kinds of input data, either a video file (e.g. .mpg) or
 be optionally accompanied with a CSV file containing video annotations. Example input sequences are available at
 https://viame.kitware.com/girder#collections.
 
+DIVE can also accept a flat Zip files with a list of images.
+
 ### Output
 
 When running an algorithmic pipelines or performing manual video annotation (and saving the annotations with the save
diff --git a/docs/Web-Version.md b/docs/Web-Version.md
index 6ceb63bae..6246bff15 100644
--- a/docs/Web-Version.md
+++ b/docs/Web-Version.md
@@ -34,6 +34,11 @@ A user account is required to store data and run pipelines on viame.kitware.com.
 * Choose a name for the data, enter the optional video playback frame rate, and press start-upload.
 * In the data browser, a new blue "Launch Annotator" button will appear next to your data.
 
+### Zip Files
+
+Single flat annotations inside of a zip file can be uploaded as well as invidiual videos or images.
+Images can be zipped together with their annotations and uploaded.
+
 !!! info
 
     All video uploaded to the web server will be transcoded as `mp4/h264`.
diff --git a/server/tests/integration/conftest.py b/server/tests/integration/conftest.py
index 873275219..53ca2218d 100644
--- a/server/tests/integration/conftest.py
+++ b/server/tests/integration/conftest.py
@@ -146,6 +146,32 @@
     },
 }
 
+zipUser: Dict[str, Dict[str, Any]] = {
+    "zipUser": {
+        'login': 'zipUser',
+        'email': 'zipUser@locahost.lan',
+        'firstName': 'zip',
+        'lastName': 'User',
+        'password': 'zipUserPass',
+        'data': [
+            {
+                'name': 'testImageZip',
+                'path': 'zipTestFiles/testImageZip.zip',
+                'fps': 1,
+                'type': 'image-sequence',
+                'trackCount': 197,
+            },
+            {
+                'name': 'testVideoZip',
+                'path': 'zipTestFiles/testVideoZip.zip',
+                'fps': 29.97002997002997,
+                'originalFps': 30000 / 1001,
+                'type': 'video',
+            },
+        ],
+    }
+}
+
 
 def getClient(name: str) -> GirderClient:
     gc = GirderClient(apiUrl='http://localhost:8010/api/v1')
diff --git a/server/tests/integration/test_zip_uploads.py b/server/tests/integration/test_zip_uploads.py
new file mode 100644
index 000000000..205c0d18f
--- /dev/null
+++ b/server/tests/integration/test_zip_uploads.py
@@ -0,0 +1,66 @@
+from pathlib import Path
+
+from girder_client import GirderClient, HttpError
+import pytest
+
+from .conftest import getClient, getTestFolder, localDataRoot, users, wait_for_jobs, zipUser
+
+
+@pytest.mark.integration
+@pytest.mark.parametrize("user", zipUser.values())
+@pytest.mark.run(order=4)
+def test_user_creation(admin_client: GirderClient, user: dict):
+    try:
+        admin_client.createUser(
+            user['login'],
+            user['email'],
+            user['firstName'],
+            user['lastName'],
+            user['password'],
+        )
+    except HttpError as err:
+        if err.response.json()['message'] != 'That login is already registered.':
+            raise err
+
+
+@pytest.mark.integration
+@pytest.mark.parametrize("user", zipUser.values())
+@pytest.mark.run(order=4)
+def test_reset_integration_env(user: dict):
+    client = GirderClient(apiUrl='http://localhost:8010/api/v1')
+    client.authenticate(username=user['login'], password=user['password'])
+    privateFolder = getTestFolder(client)
+    client.delete(f"folder/{privateFolder['_id']}")
+
+
+@pytest.mark.integration
+@pytest.mark.parametrize("user", zipUser.values())
+@pytest.mark.run(order=4)
+def test_upload_zip_data(user: dict):
+    client = GirderClient(apiUrl='http://localhost:8010/api/v1')
+    client.authenticate(username=user['login'], password=user['password'])
+
+    createdDatasets = []
+    for dataset in user['data']:
+        dsPath = localDataRoot / str(dataset['path'])
+        privateFolder = getTestFolder(client)
+        newDatasetFolder = client.createFolder(
+            privateFolder['_id'],
+            dataset['name'],
+            metadata={
+                'fps': dataset['fps'],
+                'type': dataset['type'],
+            },
+        )
+        createdDatasets.append(newDatasetFolder)
+        if Path(dsPath).is_file():
+            client.uploadFileToFolder(newDatasetFolder['_id'], str(dsPath))
+        client.post(f'dive_rpc/postprocess/{newDatasetFolder["_id"]}')
+    wait_for_jobs(client)
+    # Confirm that the new dataset looks like it should.
+    for created, expected in zip(createdDatasets, user['data']):
+        created = client.get(f'dive_dataset/{created["_id"]}')
+        if expected['type'] == 'video':
+            assert created['fps'] == expected['originalFps'] or created['fps'] == expected['fps']
+            assert created['annotate']
+            assert created['originalFps'] == expected['originalFps']

From 5c3ccb4519a77509ab9be24cc8596cb79ce54308 Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Tue, 23 Nov 2021 09:08:02 -0500
Subject: [PATCH 05/16] updating tests

---
 server/tests/integration/conftest.py          | 29 +++++++++++++++++--
 .../integration/test_download_extract.py      |  1 +
 server/tests/integration/test_zip_uploads.py  | 16 +++++-----
 3 files changed, 34 insertions(+), 12 deletions(-)

diff --git a/server/tests/integration/conftest.py b/server/tests/integration/conftest.py
index 53ca2218d..7ad934295 100644
--- a/server/tests/integration/conftest.py
+++ b/server/tests/integration/conftest.py
@@ -160,6 +160,7 @@
                 'fps': 1,
                 'type': 'image-sequence',
                 'trackCount': 197,
+                'job_status': 3,
             },
             {
                 'name': 'testVideoZip',
@@ -167,6 +168,28 @@
                 'fps': 29.97002997002997,
                 'originalFps': 30000 / 1001,
                 'type': 'video',
+                'job_status': 3,
+            },
+            {
+                'name': 'badFormatZip',
+                'path': 'zipTestFiles/badFormatZip.zip',
+                'fps': 1,
+                'type': 'image-sequence',
+                'job_status': 4,
+            },
+            {
+                'name': 'nestedZip',
+                'path': 'zipTestFiles/nestedZip.zip',
+                'fps': 1,
+                'type': 'image-sequence',
+                'job_status': 4,
+            },
+            {
+                'name': 'zipBomb',
+                'path': 'zipTestFiles/zipBomb.zip',
+                'fps': 1,
+                'type': 'image-sequence',
+                'job_status': 4,
             },
         ],
     }
@@ -192,7 +215,7 @@ def admin_client() -> GirderClient:
     return gc
 
 
-def wait_for_jobs(client: GirderClient, max_wait_timeout=30):
+def wait_for_jobs(client: GirderClient, max_wait_timeout=30, expected_status=3):
     """Wait for all worker jobs to complete"""
     start_time = time.time()
     incompleteJobs = []
@@ -218,8 +241,8 @@ def wait_for_jobs(client: GirderClient, max_wait_timeout=30):
             'limit': 1,
         },
     )
-    if len(lastJob) > 0 and lastJob[0]['status'] != 3:
-        raise Exception("Some jobs did not succeed")
+    if len(lastJob) > 0 and lastJob[0]['status'] != expected_status:
+        raise Exception(f"Some jobs did not meet their expected status: {expected_status}")
 
 
 def match_user_server_data(user: Dict[str, Any], dataset) -> List[dict]:
diff --git a/server/tests/integration/test_download_extract.py b/server/tests/integration/test_download_extract.py
index 7322b3cdf..9a16c1b26 100644
--- a/server/tests/integration/test_download_extract.py
+++ b/server/tests/integration/test_download_extract.py
@@ -11,6 +11,7 @@
     ('607ded8b1d59f90549235c3d', 'stereo.zip'),
     ('6006dfb9f751cc24d135ec29', 'TestData.zip'),
     ('61646cf37c93704c9952e84d', 'kwcoco.zip'),
+    ('619cf1f87a958c6b1f255c2c', 'zipTestFiles.zip'),
 ]
 
 
diff --git a/server/tests/integration/test_zip_uploads.py b/server/tests/integration/test_zip_uploads.py
index 205c0d18f..73dc16cb9 100644
--- a/server/tests/integration/test_zip_uploads.py
+++ b/server/tests/integration/test_zip_uploads.py
@@ -3,7 +3,7 @@
 from girder_client import GirderClient, HttpError
 import pytest
 
-from .conftest import getClient, getTestFolder, localDataRoot, users, wait_for_jobs, zipUser
+from .conftest import getTestFolder, localDataRoot, wait_for_jobs, zipUser
 
 
 @pytest.mark.integration
@@ -56,11 +56,9 @@ def test_upload_zip_data(user: dict):
         if Path(dsPath).is_file():
             client.uploadFileToFolder(newDatasetFolder['_id'], str(dsPath))
         client.post(f'dive_rpc/postprocess/{newDatasetFolder["_id"]}')
-    wait_for_jobs(client)
-    # Confirm that the new dataset looks like it should.
-    for created, expected in zip(createdDatasets, user['data']):
-        created = client.get(f'dive_dataset/{created["_id"]}')
-        if expected['type'] == 'video':
-            assert created['fps'] == expected['originalFps'] or created['fps'] == expected['fps']
-            assert created['annotate']
-            assert created['originalFps'] == expected['originalFps']
+        try:
+            wait_for_jobs(client, max_wait_timeout=30, expected_status=dataset['job_status'])
+        except Exception as ex:
+            if dataset['job_status'] == 4:
+                continue
+            raise ex

From fd98e601f91c27546f21420ba95067846bde73bb Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Mon, 29 Nov 2021 08:48:02 -0500
Subject: [PATCH 06/16] source folder

---
 server/dive_server/crud.py     | 4 ++++
 server/dive_server/crud_rpc.py | 7 ++++++-
 server/dive_tasks/tasks.py     | 3 +--
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/server/dive_server/crud.py b/server/dive_server/crud.py
index 6c69a22e5..83498e371 100644
--- a/server/dive_server/crud.py
+++ b/server/dive_server/crud.py
@@ -111,6 +111,10 @@ def move_existing_result_to_auxiliary_folder(folder, user):
         Item().move(item, auxiliary)
 
 
+def get_or_create_source_folder(folder, user):
+    return Folder().createFolder(folder, "source", reuseExisting=True, creator=user)
+
+
 def itemIsWebsafeVideo(item: Item) -> bool:
     return fromMeta(item, "codec") == "h264"
 
diff --git a/server/dive_server/crud_rpc.py b/server/dive_server/crud_rpc.py
index 1274965a8..42d944e78 100644
--- a/server/dive_server/crud_rpc.py
+++ b/server/dive_server/crud_rpc.py
@@ -393,10 +393,15 @@ def postprocess(
             dsFolder,
             filters={
                 "lowerName": {"$regex": constants.zipRegex},
-                f"meta.{constants.ZipFileExtractedMarker}": {'$exists': False},
             },
         )
+        # only support one zip file found
         for item in zipItems:
+            # if item is already extracted create/move into 'source' folder
+            if constants.ZipFileExtractedMarker in item['meta'].keys():
+                source_folder = crud.get_or_create_source_folder(dsFolder, user)
+                Item().move(item, source_folder)
+                continue
             newjob = tasks.extract_zip.apply_async(
                 queue=_get_queue_name(user),
                 kwargs=dict(
diff --git a/server/dive_tasks/tasks.py b/server/dive_tasks/tasks.py
index 0af348204..80a2aed16 100644
--- a/server/dive_tasks/tasks.py
+++ b/server/dive_tasks/tasks.py
@@ -597,8 +597,7 @@ def extract_zip(self: Task, folderId: str, itemId: str):
         validation = gc.sendRestRequest(
             'POST', '/dive_dataset/validate_files', json=listOfFileNames
         )
-        print(validation)
-        if validation:
+        if 'ok' in validation.keys() and validation['ok'] is True:
             manager.write(f"Annotations: {validation['annotations']}\n")
             manager.write(f"Media: {validation['media']}\n")
             dataset_type = validation['type']

From eb62e6ae3832f61227077faed4df12b3beab4f1f Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Mon, 29 Nov 2021 15:42:45 -0500
Subject: [PATCH 07/16] starting to address comments

---
 client/platform/web-girder/views/Upload.vue  | 35 ++++++++++----------
 server/dive_server/crud_rpc.py               |  8 ++---
 server/dive_tasks/tasks.py                   | 20 ++++++++---
 server/dive_utils/constants.py               |  2 ++
 server/tests/integration/conftest.py         | 13 ++++----
 server/tests/integration/test_zip_uploads.py |  3 +-
 6 files changed, 46 insertions(+), 35 deletions(-)

diff --git a/client/platform/web-girder/views/Upload.vue b/client/platform/web-girder/views/Upload.vue
index 2d5383106..39bddaa1c 100644
--- a/client/platform/web-girder/views/Upload.vue
+++ b/client/platform/web-girder/views/Upload.vue
@@ -70,6 +70,23 @@ export default defineComponent({
     const importMultiCamDialog = ref(false);
     const girderUpload: Ref<null | GirderUpload> = ref(null);
     const { prompt } = usePrompt();
+
+    const addPendingZipUpload = (name: string, allFiles: File[]) => {
+      const fps = clientSettings.annotationFPS || DefaultVideoFPS;
+      const defaultFilename = allFiles.length ? allFiles[0].name.replace(/\..*/, '') : 'Zip Upload';
+      pendingUploads.value.push({
+        createSubFolders: false,
+        name: defaultFilename,
+        files: [], //Will be set in the GirderUpload Component
+        meta: null,
+        annotationFile: null,
+        mediaList: allFiles,
+        type: 'zip',
+        fps,
+        uploading: false,
+      });
+    };
+
     /**
      * Initial opening of file dialog
      */
@@ -90,7 +107,6 @@ export default defineComponent({
                 processed.annotationFile, processed.mediaList,
               );
             } else {
-              // eslint-disable-next-line @typescript-eslint/no-use-before-define
               addPendingZipUpload(name, processed.fullList);
             }
           } catch (err) {
@@ -232,21 +248,6 @@ export default defineComponent({
     const getFilenameInputValue = (pendingUpload: PendingUpload) => (
       pendingUpload.createSubFolders && pendingUpload.type !== 'zip' ? 'default' : pendingUpload.name
     );
-    const addPendingZipUpload = (name: string, allFiles: File[]) => {
-      const fps = clientSettings.annotationFPS || DefaultVideoFPS;
-      const defaultFilename = allFiles.length ? allFiles[0].name.replace(/\..*/, '') : 'Zip Upload';
-      pendingUploads.value.push({
-        createSubFolders: false,
-        name: defaultFilename,
-        files: [], //Will be set in the GirderUpload Component
-        meta: null,
-        annotationFile: null,
-        mediaList: allFiles,
-        type: 'zip',
-        fps,
-        uploading: false,
-      });
-    };
     const addPendingUpload = async (
       name: string,
       allFiles: File[],
@@ -463,7 +464,7 @@ export default defineComponent({
                 </v-btn>
               </v-col>
             </v-row>
-            <v-row v-if="!pendingUpload.createSubFolders && !pendingUpload.type == 'zip'">
+            <v-row v-if="!pendingUpload.createSubFolders && pendingUpload.type != 'zip'">
               <v-col class="py-0">
                 <v-row>
                   <v-col>
diff --git a/server/dive_server/crud_rpc.py b/server/dive_server/crud_rpc.py
index 42d944e78..0d9a54fa7 100644
--- a/server/dive_server/crud_rpc.py
+++ b/server/dive_server/crud_rpc.py
@@ -393,15 +393,10 @@ def postprocess(
             dsFolder,
             filters={
                 "lowerName": {"$regex": constants.zipRegex},
+                f"meta.{constants.ZipFileExtractedMarker}": {'$exists': False},
             },
         )
-        # only support one zip file found
         for item in zipItems:
-            # if item is already extracted create/move into 'source' folder
-            if constants.ZipFileExtractedMarker in item['meta'].keys():
-                source_folder = crud.get_or_create_source_folder(dsFolder, user)
-                Item().move(item, source_folder)
-                continue
             newjob = tasks.extract_zip.apply_async(
                 queue=_get_queue_name(user),
                 kwargs=dict(
@@ -414,6 +409,7 @@ def postprocess(
             )
             newjob.job[constants.JOBCONST_PRIVATE_QUEUE] = job_is_private
             Job().save(newjob.job)
+            # only support first zip file found
             return dsFolder
 
         # transcode VIDEO if necessary
diff --git a/server/dive_tasks/tasks.py b/server/dive_tasks/tasks.py
index 80a2aed16..bab677ed6 100644
--- a/server/dive_tasks/tasks.py
+++ b/server/dive_tasks/tasks.py
@@ -588,7 +588,7 @@ def extract_zip(self: Task, folderId: str, itemId: str):
 
             for fileName in listOfFileNames:
                 if fileName.endswith('.zip'):
-                    manager.write("Nested Zip files are invalid!!!!\n")
+                    manager.write("Nested Zip files are invalid\n")
                     manager.updateStatus(JobStatus.ERROR)
                     return
                 manager.write(f"Extracting: {fileName}\n")
@@ -597,18 +597,28 @@ def extract_zip(self: Task, folderId: str, itemId: str):
         validation = gc.sendRestRequest(
             'POST', '/dive_dataset/validate_files', json=listOfFileNames
         )
-        if 'ok' in validation.keys() and validation['ok'] is True:
+        if validation.get('ok', False):
             manager.write(f"Annotations: {validation['annotations']}\n")
             manager.write(f"Media: {validation['media']}\n")
             dataset_type = validation['type']
             manager.write(f"Type: {dataset_type}\n")
-            if "ok" in validation and validation["ok"] is False:
-                manager.write(f"Message: {validation['message']}\n")
-                manager.updateStatus(JobStatus.ERROR)
             # Upload all resulting items back into the root folder
             manager.updateStatus(JobStatus.PUSHING_OUTPUT)
             gc.upload(f'{_working_directory}/**/*', folderId)
             gc.addMetadataToFolder(str(folderId), {constants.TypeMarker: dataset_type})
             gc.addMetadataToItem(str(itemId), {constants.ZipFileExtractedMarker: True})
+            # create a source folder to place the zipFile inside of
+            metadata = {constants.ZipFileExtractedMarker: True}
+            created_folder = gc.createFolder(
+                folderId, constants.SourceFolderName, reuseExisting=True, metadata=metadata
+            )
+            gc.sendRestRequest(
+                "PUT",
+                f"/item/{str(itemId)}?name={item['name']}&folderId={str(created_folder['_id'])}",
+                json=metadata,
+            )
             # After uploading the default files we do a the postprocess for video conversion now
             gc.sendRestRequest("POST", f"/dive_rpc/postprocess/{str(folderId)}")
+        else:
+            manager.write(f"Message: {validation['message']}\n")
+            manager.updateStatus(JobStatus.ERROR)
diff --git a/server/dive_utils/constants.py b/server/dive_utils/constants.py
index ef0518e3a..9cd7a354f 100644
--- a/server/dive_utils/constants.py
+++ b/server/dive_utils/constants.py
@@ -86,6 +86,8 @@
 ViameDataFolderName = "VIAME"
 # The name of the subfolder for training results
 TrainingOutputFolderName = "VIAME Training Results"
+# The name of the source folder holding zip backups
+SourceFolderName = "source"
 
 # job constants
 JOBCONST_DATASET_ID = 'datset_id'
diff --git a/server/tests/integration/conftest.py b/server/tests/integration/conftest.py
index 7ad934295..d9011e3df 100644
--- a/server/tests/integration/conftest.py
+++ b/server/tests/integration/conftest.py
@@ -4,6 +4,7 @@
 from typing import Any, Dict, List
 
 from girder_client import GirderClient
+from girder_worker.utils import JobStatus
 import pytest
 
 localDataRoot = Path('tests/integration/data')
@@ -160,7 +161,7 @@
                 'fps': 1,
                 'type': 'image-sequence',
                 'trackCount': 197,
-                'job_status': 3,
+                'job_status': JobStatus.SUCCESS,
             },
             {
                 'name': 'testVideoZip',
@@ -168,28 +169,28 @@
                 'fps': 29.97002997002997,
                 'originalFps': 30000 / 1001,
                 'type': 'video',
-                'job_status': 3,
+                'job_status': JobStatus.SUCCESS,
             },
             {
                 'name': 'badFormatZip',
                 'path': 'zipTestFiles/badFormatZip.zip',
                 'fps': 1,
                 'type': 'image-sequence',
-                'job_status': 4,
+                'job_status': JobStatus.ERROR,
             },
             {
                 'name': 'nestedZip',
                 'path': 'zipTestFiles/nestedZip.zip',
                 'fps': 1,
                 'type': 'image-sequence',
-                'job_status': 4,
+                'job_status': JobStatus.ERROR,
             },
             {
                 'name': 'zipBomb',
                 'path': 'zipTestFiles/zipBomb.zip',
                 'fps': 1,
                 'type': 'image-sequence',
-                'job_status': 4,
+                'job_status': JobStatus.ERROR,
             },
         ],
     }
@@ -215,7 +216,7 @@ def admin_client() -> GirderClient:
     return gc
 
 
-def wait_for_jobs(client: GirderClient, max_wait_timeout=30, expected_status=3):
+def wait_for_jobs(client: GirderClient, max_wait_timeout=30, expected_status=JobStatus.SUCCESS):
     """Wait for all worker jobs to complete"""
     start_time = time.time()
     incompleteJobs = []
diff --git a/server/tests/integration/test_zip_uploads.py b/server/tests/integration/test_zip_uploads.py
index 73dc16cb9..0be7baa0b 100644
--- a/server/tests/integration/test_zip_uploads.py
+++ b/server/tests/integration/test_zip_uploads.py
@@ -1,6 +1,7 @@
 from pathlib import Path
 
 from girder_client import GirderClient, HttpError
+from girder_worker.utils import JobStatus
 import pytest
 
 from .conftest import getTestFolder, localDataRoot, wait_for_jobs, zipUser
@@ -59,6 +60,6 @@ def test_upload_zip_data(user: dict):
         try:
             wait_for_jobs(client, max_wait_timeout=30, expected_status=dataset['job_status'])
         except Exception as ex:
-            if dataset['job_status'] == 4:
+            if dataset['job_status'] == JobStatus.ERROR:
                 continue
             raise ex

From a3486a6b00fb63ecca1635f8516bb9a6f33caf14 Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Wed, 1 Dec 2021 09:28:42 -0500
Subject: [PATCH 08/16] minor fixes

---
 client/platform/web-girder/views/Upload.vue | 2 +-
 server/dive_tasks/tasks.py                  | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/client/platform/web-girder/views/Upload.vue b/client/platform/web-girder/views/Upload.vue
index 39bddaa1c..de7959df3 100644
--- a/client/platform/web-girder/views/Upload.vue
+++ b/client/platform/web-girder/views/Upload.vue
@@ -464,7 +464,7 @@ export default defineComponent({
                 </v-btn>
               </v-col>
             </v-row>
-            <v-row v-if="!pendingUpload.createSubFolders && pendingUpload.type != 'zip'">
+            <v-row v-if="!pendingUpload.createSubFolders && pendingUpload.type !== 'zip'">
               <v-col class="py-0">
                 <v-row>
                   <v-col>
diff --git a/server/dive_tasks/tasks.py b/server/dive_tasks/tasks.py
index bab677ed6..bf4d03de9 100644
--- a/server/dive_tasks/tasks.py
+++ b/server/dive_tasks/tasks.py
@@ -610,7 +610,9 @@ def extract_zip(self: Task, folderId: str, itemId: str):
             # create a source folder to place the zipFile inside of
             metadata = {constants.ZipFileExtractedMarker: True}
             created_folder = gc.createFolder(
-                folderId, constants.SourceFolderName, reuseExisting=True, metadata=metadata
+                folderId,
+                constants.SourceFolderName,
+                reuseExisting=True,
             )
             gc.sendRestRequest(
                 "PUT",

From 3d7af101c4c8eb55c5436c6e313077853c7de7d7 Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Wed, 1 Dec 2021 15:04:42 -0500
Subject: [PATCH 09/16] removing zipmarker

---
 client/platform/web-girder/views/Upload.vue | 130 ++++++++++----------
 server/dive_server/crud_rpc.py              |   5 +-
 server/dive_tasks/tasks.py                  |   7 +-
 server/dive_utils/constants.py              |   1 -
 4 files changed, 67 insertions(+), 76 deletions(-)

diff --git a/client/platform/web-girder/views/Upload.vue b/client/platform/web-girder/views/Upload.vue
index de7959df3..511733ab9 100644
--- a/client/platform/web-girder/views/Upload.vue
+++ b/client/platform/web-girder/views/Upload.vue
@@ -87,33 +87,46 @@ export default defineComponent({
       });
     };
 
-    /**
-     * Initial opening of file dialog
-     */
-    const openImport = async (dstype: DatasetType | 'zip') => {
-      const ret = await openFromDisk(dstype);
-      if (!ret.canceled && ret.fileList) {
-        // eslint-disable-next-line @typescript-eslint/no-use-before-define
-        const processed = processImport(ret);
-        if (processed?.fullList?.length === 0) return;
-        if (processed && processed.fullList) {
-          const name = processed.fullList.length === 1 ? processed.fullList[0].name : '';
-          preUploadErrorMessage.value = null;
-          try {
-            if (dstype !== 'zip') {
-            // eslint-disable-next-line @typescript-eslint/no-use-before-define
-              await addPendingUpload(
-                name, processed.fullList, processed.metaFile,
-                processed.annotationFile, processed.mediaList,
-              );
-            } else {
-              addPendingZipUpload(name, processed.fullList);
-            }
-          } catch (err) {
-            preUploadErrorMessage.value = err.response?.data?.message || err;
-          }
+    const addPendingUpload = async (
+      name: string,
+      allFiles: File[],
+      meta: File | null,
+      annotationFile: File | null,
+      mediaList: File[],
+    ) => {
+      const resp = (await validateUploadGroup(allFiles.map((f) => f.name))).data;
+      if (!resp.ok) {
+        if (resp.message) {
+          preUploadErrorMessage.value = resp.message;
+        }
+        throw new Error(resp.message);
+      }
+      const fps = clientSettings.annotationFPS || DefaultVideoFPS;
+      const defaultFilename = resp.media[0];
+      const validFiles = resp.media.concat(resp.annotations);
+      // mapping needs to be done for the mixin upload functions
+      const internalFiles = allFiles
+        .filter((f) => validFiles.includes(f.name));
+      let createSubFolders = false;
+      if (resp.type === 'video') {
+        if (resp.media.length > 1) {
+          createSubFolders = true;
         }
       }
+      pendingUploads.value.push({
+        createSubFolders,
+        name:
+          internalFiles.length > 1
+            ? defaultFilename.replace(/\..*/, '')
+            : defaultFilename,
+        files: [], //Will be set in the GirderUpload Component
+        meta,
+        annotationFile,
+        mediaList,
+        type: resp.type,
+        fps,
+        uploading: false,
+      });
     };
     /**
      * Processes the imported media files to distinguish between
@@ -178,6 +191,32 @@ export default defineComponent({
       }
       return output;
     };
+    /**
+     * Initial opening of file dialog
+     */
+    const openImport = async (dstype: DatasetType | 'zip') => {
+      const ret = await openFromDisk(dstype);
+      if (!ret.canceled && ret.fileList) {
+        const processed = processImport(ret);
+        if (processed?.fullList?.length === 0) return;
+        if (processed && processed.fullList) {
+          const name = processed.fullList.length === 1 ? processed.fullList[0].name : '';
+          preUploadErrorMessage.value = null;
+          try {
+            if (dstype !== 'zip') {
+              await addPendingUpload(
+                name, processed.fullList, processed.metaFile,
+                processed.annotationFile, processed.mediaList,
+              );
+            } else {
+              addPendingZipUpload(name, processed.fullList);
+            }
+          } catch (err) {
+            preUploadErrorMessage.value = err.response?.data?.message || err;
+          }
+        }
+      }
+    };
     const openMultiCamDialog = (args: { stereo: boolean; openType: 'image-sequence' | 'video' }) => {
       stereo.value = args.stereo;
       multiCamOpenType.value = args.openType;
@@ -248,47 +287,6 @@ export default defineComponent({
     const getFilenameInputValue = (pendingUpload: PendingUpload) => (
       pendingUpload.createSubFolders && pendingUpload.type !== 'zip' ? 'default' : pendingUpload.name
     );
-    const addPendingUpload = async (
-      name: string,
-      allFiles: File[],
-      meta: File | null,
-      annotationFile: File | null,
-      mediaList: File[],
-    ) => {
-      const resp = (await validateUploadGroup(allFiles.map((f) => f.name))).data;
-      if (!resp.ok) {
-        if (resp.message) {
-          preUploadErrorMessage.value = resp.message;
-        }
-        throw new Error(resp.message);
-      }
-      const fps = clientSettings.annotationFPS || DefaultVideoFPS;
-      const defaultFilename = resp.media[0];
-      const validFiles = resp.media.concat(resp.annotations);
-      // mapping needs to be done for the mixin upload functions
-      const internalFiles = allFiles
-        .filter((f) => validFiles.includes(f.name));
-      let createSubFolders = false;
-      if (resp.type === 'video') {
-        if (resp.media.length > 1) {
-          createSubFolders = true;
-        }
-      }
-      pendingUploads.value.push({
-        createSubFolders,
-        name:
-          internalFiles.length > 1
-            ? defaultFilename.replace(/\..*/, '')
-            : defaultFilename,
-        files: [], //Will be set in the GirderUpload Component
-        meta,
-        annotationFile,
-        mediaList,
-        type: resp.type,
-        fps,
-        uploading: false,
-      });
-    };
     const remove = (pendingUpload: PendingUpload) => {
       const index = pendingUploads.value.indexOf(pendingUpload);
       pendingUploads.value.splice(index, 1);
diff --git a/server/dive_server/crud_rpc.py b/server/dive_server/crud_rpc.py
index 0d9a54fa7..421612835 100644
--- a/server/dive_server/crud_rpc.py
+++ b/server/dive_server/crud_rpc.py
@@ -391,10 +391,7 @@ def postprocess(
         # extract ZIP Files if not already completed
         zipItems = Folder().childItems(
             dsFolder,
-            filters={
-                "lowerName": {"$regex": constants.zipRegex},
-                f"meta.{constants.ZipFileExtractedMarker}": {'$exists': False},
-            },
+            filters={"lowerName": {"$regex": constants.zipRegex}},
         )
         for item in zipItems:
             newjob = tasks.extract_zip.apply_async(
diff --git a/server/dive_tasks/tasks.py b/server/dive_tasks/tasks.py
index bf4d03de9..d96a862e2 100644
--- a/server/dive_tasks/tasks.py
+++ b/server/dive_tasks/tasks.py
@@ -604,11 +604,7 @@ def extract_zip(self: Task, folderId: str, itemId: str):
             manager.write(f"Type: {dataset_type}\n")
             # Upload all resulting items back into the root folder
             manager.updateStatus(JobStatus.PUSHING_OUTPUT)
-            gc.upload(f'{_working_directory}/**/*', folderId)
-            gc.addMetadataToFolder(str(folderId), {constants.TypeMarker: dataset_type})
-            gc.addMetadataToItem(str(itemId), {constants.ZipFileExtractedMarker: True})
             # create a source folder to place the zipFile inside of
-            metadata = {constants.ZipFileExtractedMarker: True}
             created_folder = gc.createFolder(
                 folderId,
                 constants.SourceFolderName,
@@ -617,8 +613,9 @@ def extract_zip(self: Task, folderId: str, itemId: str):
             gc.sendRestRequest(
                 "PUT",
                 f"/item/{str(itemId)}?name={item['name']}&folderId={str(created_folder['_id'])}",
-                json=metadata,
             )
+            gc.upload(f'{_working_directory}/**/*', folderId)
+            gc.addMetadataToFolder(str(folderId), {constants.TypeMarker: dataset_type})
             # After uploading the default files we do a the postprocess for video conversion now
             gc.sendRestRequest("POST", f"/dive_rpc/postprocess/{str(folderId)}")
         else:
diff --git a/server/dive_utils/constants.py b/server/dive_utils/constants.py
index 9cd7a354f..adaa0f22a 100644
--- a/server/dive_utils/constants.py
+++ b/server/dive_utils/constants.py
@@ -76,7 +76,6 @@
 OriginalFPSMarker = "originalFps"
 OriginalFPSStringMarker = "originalFpsString"
 ConfidenceFiltersMarker = "confidenceFilters"
-ZipFileExtractedMarker = "extractedZip"
 
 # Other constants
 TrainedPipelineCategory = "trained"

From 7946123e8db0b382867f939fbefb2071a513010d Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Fri, 10 Dec 2021 14:21:50 -0500
Subject: [PATCH 10/16] adding in multi-import zip  and dataset import zip

---
 server/dive_tasks/tasks.py |  49 +++++++--------
 server/dive_tasks/utils.py | 124 +++++++++++++++++++++++++++++++++++++
 2 files changed, 147 insertions(+), 26 deletions(-)

diff --git a/server/dive_tasks/tasks.py b/server/dive_tasks/tasks.py
index d96a862e2..6422f8aea 100644
--- a/server/dive_tasks/tasks.py
+++ b/server/dive_tasks/tasks.py
@@ -573,6 +573,7 @@ def extract_zip(self: Task, folderId: str, itemId: str):
         file_name = str(_working_directory_path / item['name'])
         manager.write(f'Fetching input from {itemId} to {file_name}...\n')
         gc.downloadItem(itemId, _working_directory, item["name"])
+        top_level_folders = { }
         with zipfile.ZipFile(file_name, 'r') as zipObj:
             listOfFileNames = zipObj.namelist()
             sum_file_size = sum([data.file_size for data in zipObj.filelist])
@@ -587,6 +588,12 @@ def extract_zip(self: Task, folderId: str, itemId: str):
                 return
 
             for fileName in listOfFileNames:
+                if fileName.endswith('/') and fileName.count('/') == 1:
+                    top_level_folders[fileName] = 'flat'
+                if fileName.endswith('meta.json'):
+                    root_folder = fileName.replace('/meta.json', '')
+                    if root_folder in top_level_folders:
+                        top_level_folders[root_folder] = 'dataset'
                 if fileName.endswith('.zip'):
                     manager.write("Nested Zip files are invalid\n")
                     manager.updateStatus(JobStatus.ERROR)
@@ -594,30 +601,20 @@ def extract_zip(self: Task, folderId: str, itemId: str):
                 manager.write(f"Extracting: {fileName}\n")
                 zipObj.extract(fileName, f'{_working_directory}/{fileName}')
         # validation of files in folder using dive/data
-        validation = gc.sendRestRequest(
-            'POST', '/dive_dataset/validate_files', json=listOfFileNames
-        )
-        if validation.get('ok', False):
-            manager.write(f"Annotations: {validation['annotations']}\n")
-            manager.write(f"Media: {validation['media']}\n")
-            dataset_type = validation['type']
-            manager.write(f"Type: {dataset_type}\n")
-            # Upload all resulting items back into the root folder
-            manager.updateStatus(JobStatus.PUSHING_OUTPUT)
-            # create a source folder to place the zipFile inside of
-            created_folder = gc.createFolder(
-                folderId,
-                constants.SourceFolderName,
-                reuseExisting=True,
-            )
-            gc.sendRestRequest(
-                "PUT",
-                f"/item/{str(itemId)}?name={item['name']}&folderId={str(created_folder['_id'])}",
-            )
-            gc.upload(f'{_working_directory}/**/*', folderId)
-            gc.addMetadataToFolder(str(folderId), {constants.TypeMarker: dataset_type})
-            # After uploading the default files we do a the postprocess for video conversion now
-            gc.sendRestRequest("POST", f"/dive_rpc/postprocess/{str(folderId)}")
+        if bool(top_level_folders):
+            for top_folder in top_level_folders.keys():
+                if top_level_folders[top_folder] == 'flat':
+                    utils.upload_flat_media_files(gc, manager, folderId, Path(f"{_working_directory}/{top_folder}"), top_folder)
+                elif top_level_folders[top_folder] == 'dataset' and len(top_level_folders) == 1:
         else:
-            manager.write(f"Message: {validation['message']}\n")
-            manager.updateStatus(JobStatus.ERROR)
+            utils.upload_flat_media_files(gc, manager, folderId, item, _working_directory)
+
+        created_folder = gc.createFolder(
+            folderId,
+            constants.SourceFolderName,
+            reuseExisting=True,
+        )
+        gc.sendRestRequest(
+            "PUT",
+            f"/item/{str(item['_id'])}?name={item['name']}&folderId={str(created_folder['_id'])}",
+        )
diff --git a/server/dive_tasks/utils.py b/server/dive_tasks/utils.py
index bacf16d2b..5f0ee5826 100644
--- a/server/dive_tasks/utils.py
+++ b/server/dive_tasks/utils.py
@@ -1,4 +1,6 @@
 from datetime import datetime, timedelta
+import json
+import os
 from pathlib import Path
 import shutil
 import signal
@@ -13,6 +15,7 @@
 from girder_worker.utils import JobManager, JobStatus
 
 from dive_utils import constants, models
+from dive_utils.types import GirderModel
 
 TIMEOUT_COUNT = 'timeout_count'
 TIMEOUT_LAST_CHECKED = 'last_checked'
@@ -157,3 +160,124 @@ def download_source_media(girder_client: GirderClient, datasetId: str, dest: Pat
         return [destination_path]
     else:
         raise Exception(f"unexpected metadata {str(dataset.dict())}")
+
+def upload_flat_media_files(
+    gc: GirderClient,
+    manager: JobManager,
+    folderId: str,
+    working_directory: Path,
+    create_subfolder = '',
+):
+    listOfFileNames = os.listdir(working_directory)
+    validation = gc.sendRestRequest(
+        'POST', '/dive_dataset/validate_files', json=listOfFileNames
+    )
+    root_folderId = folderId
+    if validation.get('ok', False):
+        manager.write(f"Annotations: {validation['annotations']}\n")
+        manager.write(f"Media: {validation['media']}\n")
+        dataset_type = validation['type']
+        manager.write(f"Type: {dataset_type}\n")
+        if len(create_subfolder) > 0:
+            sub_folder = gc.createFolder(
+                folderId,
+                create_subfolder,
+                reuseExisting=True,
+            )
+            root_folderId = sub_folder
+
+        # Upload all resulting items back into the root folder
+        manager.updateStatus(JobStatus.PUSHING_OUTPUT)
+        # create a source folder to place the zipFile inside of
+        gc.upload(f'{working_directory}/**/*', root_folderId)
+        gc.addMetadataToFolder(str(root_folderId), {constants.TypeMarker: dataset_type})
+        # After uploading the default files we do a the postprocess for video conversion now
+        gc.sendRestRequest("POST", f"/dive_rpc/postprocess/{str(root_folderId)}")
+    else:
+        manager.write(f"Message: {validation['message']}\n")
+        manager.updateStatus(JobStatus.ERROR)
+
+
+def upload_zipped_dataset(
+    gc: GirderClient,
+    manager: JobManager,
+    folderId: str,
+    item: GirderModel,
+    working_directory: Path,
+    create_subfolder= '',
+):
+    listOfFileNames = os.listdir(working_directory)
+    if "meta.json" not in listOfFileNames:
+        manager.write("Could not find meta.json file within the subdirectroy\n")
+        return
+    # load meta.json to get datatype and verify list of files
+    meta = json.loads(f"{working_directory}/meta.json")
+    type = meta[constants.TypeMarker]
+    if type == constants.ImageSequenceType:
+        imageData = meta['imageData']
+        for item in imageData: 
+            if item["filename"] not in listOfFileNames:
+                manager.write("Could not find {item['filename']} file within the list of files\n")
+                return
+    elif type == constants.VideoType:
+        video = meta["video"]
+        if video["filename"] not in listOfFileNames:
+            manager.write("Could not find {item['filename']} file within the list of files\n")
+            return
+    if len(create_subfolder) > 0:
+        sub_folder = gc.createFolder(
+            folderId,
+            create_subfolder,
+            reuseExisting=True,
+        )
+        root_folderId = sub_folder
+        manager.updateStatus(JobStatus.PUSHING_OUTPUT)
+        # create a source folder to place the zipFile inside of
+        gc.upload(f'{working_directory}/**/*', root_folderId)
+    # Now we set all the metadata for the folders and items
+    all_files = list(gc.listItem(root_folderId))
+    root_meta = {
+        "type": type,
+        "attributes": meta["attributes"],
+        "customTypeStyling": meta["customTypeStyling"],
+        "confidenceFilters": meta["confidenceFilters"],
+        "fps": meta["fps"],
+        "version": meta["version"],
+    }
+    if type == constants.VideoType:
+        # set transcoded and non-transcoded versions
+        transcoded_video = list(gc.listItem(root_folderId, video["filename"]))
+        if len(transcoded_video) == 1:
+            ffprobe = meta["ffprobe_info"]
+            avgFpsString = ffprobe["avg_frame_rate"]
+            dividend, divisor = [int(v) for v in avgFpsString.split('/')]
+            originalFps = dividend / divisor
+
+            transcoded_metadata = {
+                "codec": "h264",
+                "originalFps": originalFps,
+                "originalFpsString": avgFpsString,
+                "source_video": False,
+                "transcoder": "ffmpeg",
+            }
+            gc.addMetadataToItem(str(transcoded_video[0]['_id']), transcoded_metadata)
+            # other video is tagged as the source video
+            for item in all_files:
+                if item["name"].endswith(constants.validVideoFormats) and item["name"] != video["filename"]:
+                    source_metadata = {
+                        "codec": ffprobe["codec"],
+                        "originalFps": originalFps, 
+                        "originalFpsString": avgFpsString,
+                        "source_video": False,
+                    }
+                    gc.addMetadataToItem(str(item['_id']), source_metadata)
+            root_meta["originalFps"] = originalFps
+            root_meta["originalFpsString"] = avgFpsString
+
+    # Need to tag folder Level data (annotate, and others)
+    root_meta[constants.DatasetMarker] = True
+    gc.addMetadataToFolder(root_folderId, root_meta)
+    # Need to make sure all detection files in auxiliary subfolder are proper tagged as well so post process doesn't double process them.
+
+
+

From 1bad7da0b66f6b0ec738637954991a26c8404776 Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Mon, 13 Dec 2021 10:34:18 -0500
Subject: [PATCH 11/16] gAdding in beginning of dataset import

---
 server/dive_tasks/tasks.py | 35 ++++++++++++++++-----
 server/dive_tasks/utils.py | 63 ++++++++++++++++++++++----------------
 2 files changed, 64 insertions(+), 34 deletions(-)

diff --git a/server/dive_tasks/tasks.py b/server/dive_tasks/tasks.py
index 6422f8aea..5fa08cbcb 100644
--- a/server/dive_tasks/tasks.py
+++ b/server/dive_tasks/tasks.py
@@ -573,7 +573,7 @@ def extract_zip(self: Task, folderId: str, itemId: str):
         file_name = str(_working_directory_path / item['name'])
         manager.write(f'Fetching input from {itemId} to {file_name}...\n')
         gc.downloadItem(itemId, _working_directory, item["name"])
-        top_level_folders = { }
+        top_level_folders = {}
         with zipfile.ZipFile(file_name, 'r') as zipObj:
             listOfFileNames = zipObj.namelist()
             sum_file_size = sum([data.file_size for data in zipObj.filelist])
@@ -589,25 +589,46 @@ def extract_zip(self: Task, folderId: str, itemId: str):
 
             for fileName in listOfFileNames:
                 if fileName.endswith('/') and fileName.count('/') == 1:
-                    top_level_folders[fileName] = 'flat'
+                    top_level_folders[fileName.replace('/', '')] = 'flat'
                 if fileName.endswith('meta.json'):
                     root_folder = fileName.replace('/meta.json', '')
-                    if root_folder in top_level_folders:
-                        top_level_folders[root_folder] = 'dataset'
+                    top_level_folders[root_folder] = 'dataset'
                 if fileName.endswith('.zip'):
                     manager.write("Nested Zip files are invalid\n")
                     manager.updateStatus(JobStatus.ERROR)
                     return
                 manager.write(f"Extracting: {fileName}\n")
-                zipObj.extract(fileName, f'{_working_directory}/{fileName}')
+                zipObj.extract(fileName, f'{_working_directory}')
         # validation of files in folder using dive/data
+        print(top_level_folders)
         if bool(top_level_folders):
             for top_folder in top_level_folders.keys():
                 if top_level_folders[top_folder] == 'flat':
-                    utils.upload_flat_media_files(gc, manager, folderId, Path(f"{_working_directory}/{top_folder}"), top_folder)
+                    utils.upload_zipped_flat_media_files(
+                        gc,
+                        manager,
+                        folderId,
+                        Path(f"{_working_directory}/{top_folder}"),
+                        top_folder,
+                    )
                 elif top_level_folders[top_folder] == 'dataset' and len(top_level_folders) == 1:
+                    utils.upload_exported_zipped_dataset(
+                        gc,
+                        manager,
+                        folderId,
+                        Path(f"{_working_directory}/{top_folder}"),
+                    )
+                elif top_level_folders[top_folder] == 'dataset':
+                    utils.upload_exported_zipped_dataset(
+                        gc,
+                        manager,
+                        folderId,
+                        Path(f"{_working_directory}/{top_folder}"),
+                        top_folder,
+                    )
+
         else:
-            utils.upload_flat_media_files(gc, manager, folderId, item, _working_directory)
+            utils.upload_zipped_flat_media_files(gc, manager, folderId, _working_directory)
 
         created_folder = gc.createFolder(
             folderId,
diff --git a/server/dive_tasks/utils.py b/server/dive_tasks/utils.py
index 5f0ee5826..33cbb4b13 100644
--- a/server/dive_tasks/utils.py
+++ b/server/dive_tasks/utils.py
@@ -161,18 +161,18 @@ def download_source_media(girder_client: GirderClient, datasetId: str, dest: Pat
     else:
         raise Exception(f"unexpected metadata {str(dataset.dict())}")
 
-def upload_flat_media_files(
+
+def upload_zipped_flat_media_files(
     gc: GirderClient,
     manager: JobManager,
     folderId: str,
     working_directory: Path,
-    create_subfolder = '',
+    create_subfolder='',
 ):
     listOfFileNames = os.listdir(working_directory)
-    validation = gc.sendRestRequest(
-        'POST', '/dive_dataset/validate_files', json=listOfFileNames
-    )
+    validation = gc.sendRestRequest('POST', '/dive_dataset/validate_files', json=listOfFileNames)
     root_folderId = folderId
+    default_fps = gc.getFolder(root_folderId).get(f"meta.{constants.FPSMarker}", -1)
     if validation.get('ok', False):
         manager.write(f"Annotations: {validation['annotations']}\n")
         manager.write(f"Media: {validation['media']}\n")
@@ -184,13 +184,16 @@ def upload_flat_media_files(
                 create_subfolder,
                 reuseExisting=True,
             )
-            root_folderId = sub_folder
+            root_folderId = str(sub_folder["_id"])
 
         # Upload all resulting items back into the root folder
         manager.updateStatus(JobStatus.PUSHING_OUTPUT)
         # create a source folder to place the zipFile inside of
-        gc.upload(f'{working_directory}/**/*', root_folderId)
-        gc.addMetadataToFolder(str(root_folderId), {constants.TypeMarker: dataset_type})
+        gc.upload(f'{working_directory}/*', root_folderId)
+        gc.addMetadataToFolder(
+            str(root_folderId),
+            {constants.TypeMarker: dataset_type, constants.FPSMarker: default_fps},
+        )
         # After uploading the default files we do a the postprocess for video conversion now
         gc.sendRestRequest("POST", f"/dive_rpc/postprocess/{str(root_folderId)}")
     else:
@@ -198,25 +201,27 @@ def upload_flat_media_files(
         manager.updateStatus(JobStatus.ERROR)
 
 
-def upload_zipped_dataset(
+def upload_exported_zipped_dataset(
     gc: GirderClient,
     manager: JobManager,
     folderId: str,
-    item: GirderModel,
     working_directory: Path,
-    create_subfolder= '',
+    create_subfolder='',
 ):
     listOfFileNames = os.listdir(working_directory)
     if "meta.json" not in listOfFileNames:
         manager.write("Could not find meta.json file within the subdirectroy\n")
         return
+    print(listOfFileNames)
     # load meta.json to get datatype and verify list of files
-    meta = json.loads(f"{working_directory}/meta.json")
+    meta = {}
+    with open(f"{working_directory}/meta.json") as f:
+        meta = json.load(f)
     type = meta[constants.TypeMarker]
     if type == constants.ImageSequenceType:
         imageData = meta['imageData']
-        for item in imageData: 
-            if item["filename"] not in listOfFileNames:
+        for image in imageData:
+            if image["filename"] not in listOfFileNames:
                 manager.write("Could not find {item['filename']} file within the list of files\n")
                 return
     elif type == constants.VideoType:
@@ -224,29 +229,33 @@ def upload_zipped_dataset(
         if video["filename"] not in listOfFileNames:
             manager.write("Could not find {item['filename']} file within the list of files\n")
             return
+    # remove the auxilary directory so we don't have to tag them all
+    if "auxiliary" in listOfFileNames and os.path.isdir(f'{working_directory}/auxiliary'):
+        shutil.rmtree(f'{working_directory}/auxiliary')
+    root_folderId = folderId
     if len(create_subfolder) > 0:
         sub_folder = gc.createFolder(
             folderId,
             create_subfolder,
             reuseExisting=True,
         )
-        root_folderId = sub_folder
+        root_folderId = str(sub_folder['_id'])
         manager.updateStatus(JobStatus.PUSHING_OUTPUT)
         # create a source folder to place the zipFile inside of
-        gc.upload(f'{working_directory}/**/*', root_folderId)
+    gc.upload(f'{working_directory}/*', root_folderId)
     # Now we set all the metadata for the folders and items
     all_files = list(gc.listItem(root_folderId))
     root_meta = {
         "type": type,
-        "attributes": meta["attributes"],
-        "customTypeStyling": meta["customTypeStyling"],
-        "confidenceFilters": meta["confidenceFilters"],
+        "attributes": meta.get("attributes", None),
+        "customTypeStyling": meta.get("customTypeStyling", None),
+        "confidenceFilters": meta.get("confidenceFilters", None),
         "fps": meta["fps"],
         "version": meta["version"],
     }
     if type == constants.VideoType:
         # set transcoded and non-transcoded versions
-        transcoded_video = list(gc.listItem(root_folderId, video["filename"]))
+        transcoded_video = list(gc.listItem(root_folderId, name=video["filename"]))
         if len(transcoded_video) == 1:
             ffprobe = meta["ffprobe_info"]
             avgFpsString = ffprobe["avg_frame_rate"]
@@ -263,10 +272,13 @@ def upload_zipped_dataset(
             gc.addMetadataToItem(str(transcoded_video[0]['_id']), transcoded_metadata)
             # other video is tagged as the source video
             for item in all_files:
-                if item["name"].endswith(constants.validVideoFormats) and item["name"] != video["filename"]:
+                if (
+                    item["name"].endswith(tuple(constants.validVideoFormats))
+                    and item["name"] != video["filename"]
+                ):
                     source_metadata = {
-                        "codec": ffprobe["codec"],
-                        "originalFps": originalFps, 
+                        "codec": ffprobe["codec_name"],
+                        "originalFps": originalFps,
                         "originalFpsString": avgFpsString,
                         "source_video": False,
                     }
@@ -277,7 +289,4 @@ def upload_zipped_dataset(
     # Need to tag folder Level data (annotate, and others)
     root_meta[constants.DatasetMarker] = True
     gc.addMetadataToFolder(root_folderId, root_meta)
-    # Need to make sure all detection files in auxiliary subfolder are proper tagged as well so post process doesn't double process them.
-
-
-
+    gc.post(f'dive_rpc/postprocess/{root_folderId}', data={"skipJobs": True})

From 6c473b48b397a8c42bd7ebe1897e676fa6d24b14 Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Mon, 13 Dec 2021 16:18:37 -0500
Subject: [PATCH 12/16] integration testing for multi zip files

---
 server/dive_tasks/tasks.py                   | 25 ++++++-----
 server/dive_tasks/utils.py                   |  3 +-
 server/tests/integration/conftest.py         | 46 ++++++++++++++++++++
 server/tests/integration/test_zip_uploads.py |  9 ++++
 4 files changed, 70 insertions(+), 13 deletions(-)

diff --git a/server/dive_tasks/tasks.py b/server/dive_tasks/tasks.py
index 5fa08cbcb..92e6e5d69 100644
--- a/server/dive_tasks/tasks.py
+++ b/server/dive_tasks/tasks.py
@@ -599,8 +599,19 @@ def extract_zip(self: Task, folderId: str, itemId: str):
                     return
                 manager.write(f"Extracting: {fileName}\n")
                 zipObj.extract(fileName, f'{_working_directory}')
+        # remove the zip file
+        os.remove(file_name)
         # validation of files in folder using dive/data
-        print(top_level_folders)
+        created_folder = gc.createFolder(
+            folderId,
+            constants.SourceFolderName,
+            reuseExisting=True,
+        )
+        gc.sendRestRequest(
+            "PUT",
+            f"/item/{str(item['_id'])}?folderId={str(created_folder['_id'])}",
+        )
+
         if bool(top_level_folders):
             for top_folder in top_level_folders.keys():
                 if top_level_folders[top_folder] == 'flat':
@@ -628,14 +639,4 @@ def extract_zip(self: Task, folderId: str, itemId: str):
                     )
 
         else:
-            utils.upload_zipped_flat_media_files(gc, manager, folderId, _working_directory)
-
-        created_folder = gc.createFolder(
-            folderId,
-            constants.SourceFolderName,
-            reuseExisting=True,
-        )
-        gc.sendRestRequest(
-            "PUT",
-            f"/item/{str(item['_id'])}?name={item['name']}&folderId={str(created_folder['_id'])}",
-        )
+            utils.upload_zipped_flat_media_files(gc, manager, folderId, Path(_working_directory))
diff --git a/server/dive_tasks/utils.py b/server/dive_tasks/utils.py
index 33cbb4b13..2d62d14eb 100644
--- a/server/dive_tasks/utils.py
+++ b/server/dive_tasks/utils.py
@@ -15,7 +15,6 @@
 from girder_worker.utils import JobManager, JobStatus
 
 from dive_utils import constants, models
-from dive_utils.types import GirderModel
 
 TIMEOUT_COUNT = 'timeout_count'
 TIMEOUT_LAST_CHECKED = 'last_checked'
@@ -190,6 +189,8 @@ def upload_zipped_flat_media_files(
         manager.updateStatus(JobStatus.PUSHING_OUTPUT)
         # create a source folder to place the zipFile inside of
         gc.upload(f'{working_directory}/*', root_folderId)
+        if dataset_type == constants.ImageSequenceType and default_fps == -1:
+            default_fps = 1
         gc.addMetadataToFolder(
             str(root_folderId),
             {constants.TypeMarker: dataset_type, constants.FPSMarker: default_fps},
diff --git a/server/tests/integration/conftest.py b/server/tests/integration/conftest.py
index d9011e3df..fd30c39a0 100644
--- a/server/tests/integration/conftest.py
+++ b/server/tests/integration/conftest.py
@@ -171,6 +171,52 @@
                 'type': 'video',
                 'job_status': JobStatus.SUCCESS,
             },
+            {
+                'name': 'singleDatasetImport',
+                'path': 'zipTestFiles/singleDatasetImport.zip',
+                'fps': 29.97002997002997,
+                'originalFps': 30000 / 1001,
+                'type': 'video',
+                'job_status': JobStatus.SUCCESS,
+            },
+            {
+                'name': 'flatMultiImport',
+                'path': 'zipTestFiles/flatMultiImport.zip',
+                'fps': -1,
+                'type': 'zip',
+                'subDatasets': [
+                    {
+                        'name': 'testVideoZip',
+                        'type': 'video',
+                        'fps': 29.97002997002997,
+                    },
+                    {
+                        'name': 'testImageZip',
+                        'type': 'image-sequence',
+                        'fps': 1,
+                    },
+                ],
+                'job_status': JobStatus.SUCCESS,
+            },
+            {
+                'name': 'MultiDatasetImport',
+                'path': 'zipTestFiles/MultiDatasetImport.zip',
+                'fps': -1,
+                'type': 'zip',
+                'subDatasets': [
+                    {
+                        'name': 'video1_train_mp4',
+                        'type': 'video',
+                        'fps': 29.97002997002997,
+                    },
+                    {
+                        'name': 'testtrain1_imagelist',
+                        'type': 'image-sequence',
+                        'fps': 1,
+                    },
+                ],
+                'job_status': JobStatus.SUCCESS,
+            },
             {
                 'name': 'badFormatZip',
                 'path': 'zipTestFiles/badFormatZip.zip',
diff --git a/server/tests/integration/test_zip_uploads.py b/server/tests/integration/test_zip_uploads.py
index 0be7baa0b..d0bbea91e 100644
--- a/server/tests/integration/test_zip_uploads.py
+++ b/server/tests/integration/test_zip_uploads.py
@@ -63,3 +63,12 @@ def test_upload_zip_data(user: dict):
             if dataset['job_status'] == JobStatus.ERROR:
                 continue
             raise ex
+        # verify sub datasets if they exist
+        if dataset.get('subDatasets', False):
+            folders = list(client.listFolder(newDatasetFolder['_id']))
+            for item in dataset["subDatasets"]:
+                matches = [x for x in folders if x["name"] == item["name"]]
+                if len(matches) > 0:
+                    meta = matches[0].get("meta", {})
+                    assert meta.get("fps", -1) == item["fps"]
+                    assert meta.get("type", "") == item["type"]

From 4d0396ab66da053898c150799c096286907ebb4d Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Wed, 15 Dec 2021 11:33:47 -0500
Subject: [PATCH 13/16] reviewing and simplifying

---
 README.md                      |  2 --
 docs/Web-Version.md            |  9 +++++--
 server/dive_server/crud.py     |  4 +++-
 server/dive_tasks/tasks.py     | 43 ++++++++++++++++++++++++----------
 server/dive_tasks/utils.py     | 20 +++++++++-------
 server/dive_utils/constants.py |  4 ++++
 6 files changed, 56 insertions(+), 26 deletions(-)

diff --git a/README.md b/README.md
index 26f4551f2..4423e2394 100644
--- a/README.md
+++ b/README.md
@@ -37,8 +37,6 @@ DIVE takes two different kinds of input data, either a video file (e.g. .mpg) or
 be optionally accompanied with a CSV file containing video annotations. Example input sequences are available at
 https://viame.kitware.com/girder#collections.
 
-DIVE can also accept a flat Zip files with a list of images.
-
 ### Output
 
 When running an algorithmic pipelines or performing manual video annotation (and saving the annotations with the save
diff --git a/docs/Web-Version.md b/docs/Web-Version.md
index 6246bff15..128507030 100644
--- a/docs/Web-Version.md
+++ b/docs/Web-Version.md
@@ -36,8 +36,13 @@ A user account is required to store data and run pipelines on viame.kitware.com.
 
 ### Zip Files
 
-Single flat annotations inside of a zip file can be uploaded as well as invidiual videos or images.
-Images can be zipped together with their annotations and uploaded.
+A zip import can have one of the following file combinations:
+
+* One or more images, an optional annotation file, and an optional configuration file
+* One video with an optional annotation file and an optional configuration file
+* One or more folders which contain the above examples (These will be converted to separate datasets)
+* An exported zip file from DIVE
+
 
 !!! info
 
diff --git a/server/dive_server/crud.py b/server/dive_server/crud.py
index 83498e371..0255a780b 100644
--- a/server/dive_server/crud.py
+++ b/server/dive_server/crud.py
@@ -102,7 +102,9 @@ def get_static_pipelines_path() -> Path:
 
 
 def get_or_create_auxiliary_folder(folder, user):
-    return Folder().createFolder(folder, "auxiliary", reuseExisting=True, creator=user)
+    return Folder().createFolder(
+        folder, constants.AuxiliaryFolderName, reuseExisting=True, creator=user
+    )
 
 
 def move_existing_result_to_auxiliary_folder(folder, user):
diff --git a/server/dive_tasks/tasks.py b/server/dive_tasks/tasks.py
index 92e6e5d69..69874e137 100644
--- a/server/dive_tasks/tasks.py
+++ b/server/dive_tasks/tasks.py
@@ -588,10 +588,12 @@ def extract_zip(self: Task, folderId: str, itemId: str):
                 return
 
             for fileName in listOfFileNames:
+                # contains a sub folder
                 if fileName.endswith('/') and fileName.count('/') == 1:
                     top_level_folders[fileName.replace('/', '')] = 'flat'
-                if fileName.endswith('meta.json'):
-                    root_folder = fileName.replace('/meta.json', '')
+                # sub folder has a meta.json so it is imported
+                if fileName.endswith(constants.MetaFileName):
+                    root_folder = fileName.replace(f'/{constants.MetaFileName}', '')
                     top_level_folders[root_folder] = 'dataset'
                 if fileName.endswith('.zip'):
                     manager.write("Nested Zip files are invalid\n")
@@ -612,31 +614,46 @@ def extract_zip(self: Task, folderId: str, itemId: str):
             f"/item/{str(item['_id'])}?folderId={str(created_folder['_id'])}",
         )
 
-        if bool(top_level_folders):
+        if len(top_level_folders) > 0:
             for top_folder in top_level_folders.keys():
+                base_folder = ''  # default for no subfolder
+                # single flat/dataset import
+                if len(top_level_folders) != 1:
+                    base_folder = top_folder
+                # multiple flat/dataset import
                 if top_level_folders[top_folder] == 'flat':
                     utils.upload_zipped_flat_media_files(
                         gc,
                         manager,
                         folderId,
                         Path(f"{_working_directory}/{top_folder}"),
-                        top_folder,
-                    )
-                elif top_level_folders[top_folder] == 'dataset' and len(top_level_folders) == 1:
-                    utils.upload_exported_zipped_dataset(
-                        gc,
-                        manager,
-                        folderId,
-                        Path(f"{_working_directory}/{top_folder}"),
+                        base_folder,
                     )
+                # multiple flat file exports combined
                 elif top_level_folders[top_folder] == 'dataset':
                     utils.upload_exported_zipped_dataset(
                         gc,
                         manager,
                         folderId,
                         Path(f"{_working_directory}/{top_folder}"),
-                        top_folder,
+                        base_folder,
                     )
 
-        else:
+        else:  # flat zip file with media and/or annotation data inside
             utils.upload_zipped_flat_media_files(gc, manager, folderId, Path(_working_directory))
+
+        # remove auxilary folder and metadata if there are sub folders
+        if len(top_level_folders) > 1:
+            sub_folders = list(gc.listFolder(folderId))
+            aux_folders = list(
+                (x for x in sub_folders if x['name'] == constants.AuxiliaryFolderName)
+            )
+            if len(aux_folders) > 0:
+                gc.delete(f"folder/{aux_folders[0]['_id']}")
+            # remove metadata
+            metdata = [constants.TypeMarker, constants.FPSMarker, constants.DatasetMarker]
+            gc.sendRestRequest(
+                "DELETE",
+                f"folder/{folderId}/metadata",
+                json=metdata,
+            )
diff --git a/server/dive_tasks/utils.py b/server/dive_tasks/utils.py
index 2d62d14eb..9f2148952 100644
--- a/server/dive_tasks/utils.py
+++ b/server/dive_tasks/utils.py
@@ -166,8 +166,9 @@ def upload_zipped_flat_media_files(
     manager: JobManager,
     folderId: str,
     working_directory: Path,
-    create_subfolder='',
+    create_subfolder=False,
 ):
+    """Takes a flat folder of media files and/or annotation and generates a dataset from it"""
     listOfFileNames = os.listdir(working_directory)
     validation = gc.sendRestRequest('POST', '/dive_dataset/validate_files', json=listOfFileNames)
     root_folderId = folderId
@@ -177,7 +178,7 @@ def upload_zipped_flat_media_files(
         manager.write(f"Media: {validation['media']}\n")
         dataset_type = validation['type']
         manager.write(f"Type: {dataset_type}\n")
-        if len(create_subfolder) > 0:
+        if create_subfolder != '':
             sub_folder = gc.createFolder(
                 folderId,
                 create_subfolder,
@@ -209,14 +210,15 @@ def upload_exported_zipped_dataset(
     working_directory: Path,
     create_subfolder='',
 ):
+    """Uploads a folder that is generated from the export of a zip file and sets metadata"""
     listOfFileNames = os.listdir(working_directory)
-    if "meta.json" not in listOfFileNames:
-        manager.write("Could not find meta.json file within the subdirectroy\n")
+    if constants.MetaFileName not in listOfFileNames:
+        manager.write(f"Could not find {constants.MetaFileName} file within the subdirectroy\n")
         return
     print(listOfFileNames)
     # load meta.json to get datatype and verify list of files
     meta = {}
-    with open(f"{working_directory}/meta.json") as f:
+    with open(f"{working_directory}/{constants.MetaFileName}") as f:
         meta = json.load(f)
     type = meta[constants.TypeMarker]
     if type == constants.ImageSequenceType:
@@ -231,10 +233,12 @@ def upload_exported_zipped_dataset(
             manager.write("Could not find {item['filename']} file within the list of files\n")
             return
     # remove the auxilary directory so we don't have to tag them all
-    if "auxiliary" in listOfFileNames and os.path.isdir(f'{working_directory}/auxiliary'):
-        shutil.rmtree(f'{working_directory}/auxiliary')
+    if constants.AuxiliaryFolderName in listOfFileNames and os.path.isdir(
+        f'{working_directory}/{constants.AuxiliaryFolderName}'
+    ):
+        shutil.rmtree(f'{working_directory}/{constants.AuxiliaryFolderName}')
     root_folderId = folderId
-    if len(create_subfolder) > 0:
+    if create_subfolder != '':
         sub_folder = gc.createFolder(
             folderId,
             create_subfolder,
diff --git a/server/dive_utils/constants.py b/server/dive_utils/constants.py
index adaa0f22a..0ee91c535 100644
--- a/server/dive_utils/constants.py
+++ b/server/dive_utils/constants.py
@@ -87,6 +87,10 @@
 TrainingOutputFolderName = "VIAME Training Results"
 # The name of the source folder holding zip backups
 SourceFolderName = "source"
+# The name of the auxiliary folder
+AuxiliaryFolderName = "auxiliary"
+# the name of the meta file
+MetaFileName = "meta.json"
 
 # job constants
 JOBCONST_DATASET_ID = 'datset_id'

From 9a0c5658aef11d5afe7891767a3a47bfda7e59e4 Mon Sep 17 00:00:00 2001
From: BryonLewis <61746913+BryonLewis@users.noreply.github.com>
Date: Mon, 20 Dec 2021 15:01:30 -0500
Subject: [PATCH 14/16] Update docs/Web-Version.md

Co-authored-by: Brandon Davis <brandon.davis@kitware.com>
---
 docs/Web-Version.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/Web-Version.md b/docs/Web-Version.md
index 128507030..6c3c1121c 100644
--- a/docs/Web-Version.md
+++ b/docs/Web-Version.md
@@ -41,7 +41,8 @@ A zip import can have one of the following file combinations:
 * One or more images, an optional annotation file, and an optional configuration file
 * One video with an optional annotation file and an optional configuration file
 * One or more folders which contain the above examples (These will be converted to separate datasets)
-* An exported zip file from DIVE
+
+Zip import also accepts zip archive files that were generated by DIVE Web's `Download -> Everything` export button.
 
 
 !!! info

From 024d0c1c7340db4e55eef9207ea8e0545c019d1a Mon Sep 17 00:00:00 2001
From: Bryon Lewis <Bryon.Lewis@kitware.com>
Date: Mon, 20 Dec 2021 16:09:11 -0500
Subject: [PATCH 15/16] addressing comments

---
 server/dive_server/crud_rpc.py               | 15 ++++++++++-----
 server/dive_tasks/tasks.py                   | 11 ++++-------
 server/dive_tasks/utils.py                   |  2 +-
 server/tests/integration/test_zip_uploads.py |  4 ++--
 4 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/server/dive_server/crud_rpc.py b/server/dive_server/crud_rpc.py
index 421612835..21d5f26e5 100644
--- a/server/dive_server/crud_rpc.py
+++ b/server/dive_server/crud_rpc.py
@@ -389,11 +389,18 @@ def postprocess(
         token = Token().createToken(user=user, days=2)
 
         # extract ZIP Files if not already completed
-        zipItems = Folder().childItems(
-            dsFolder,
-            filters={"lowerName": {"$regex": constants.zipRegex}},
+        zipItems = list(
+            Folder().childItems(
+                dsFolder,
+                filters={"lowerName": {"$regex": constants.zipRegex}},
+            )
         )
+        if len(zipItems) > 1:
+            raise RestException('There are multiple zip files in the folder.')
         for item in zipItems:
+            total_items = len(list((Folder().childItems(dsFolder))))
+            if total_items > 1:
+                raise RestException('There are multiple files besides a zip, cannot continue')
             newjob = tasks.extract_zip.apply_async(
                 queue=_get_queue_name(user),
                 kwargs=dict(
@@ -406,8 +413,6 @@ def postprocess(
             )
             newjob.job[constants.JOBCONST_PRIVATE_QUEUE] = job_is_private
             Job().save(newjob.job)
-            # only support first zip file found
-            return dsFolder
 
         # transcode VIDEO if necessary
         videoItems = Folder().childItems(
diff --git a/server/dive_tasks/tasks.py b/server/dive_tasks/tasks.py
index 69874e137..514303c25 100644
--- a/server/dive_tasks/tasks.py
+++ b/server/dive_tasks/tasks.py
@@ -584,9 +584,7 @@ def extract_zip(self: Task, folderId: str, itemId: str):
                     f"Compression ratio is exceedingly high at {ratio}\n\
                     Please contact an admin at viame-web@kitware.com if this is a valid zip file"
                 )
-                manager.updateStatus(JobStatus.ERROR)
-                return
-
+                raise Exception("High Compression Ratio for Zip File")
             for fileName in listOfFileNames:
                 # contains a sub folder
                 if fileName.endswith('/') and fileName.count('/') == 1:
@@ -597,13 +595,12 @@ def extract_zip(self: Task, folderId: str, itemId: str):
                     top_level_folders[root_folder] = 'dataset'
                 if fileName.endswith('.zip'):
                     manager.write("Nested Zip files are invalid\n")
-                    manager.updateStatus(JobStatus.ERROR)
-                    return
+                    raise Exception("Nested Zip Files are invalid")
                 manager.write(f"Extracting: {fileName}\n")
                 zipObj.extract(fileName, f'{_working_directory}')
-        # remove the zip file
+        # remove the zip file so it isn't uploaded back to the folder
         os.remove(file_name)
-        # validation of files in folder using dive/data
+        # Create source folder and move zip file there
         created_folder = gc.createFolder(
             folderId,
             constants.SourceFolderName,
diff --git a/server/dive_tasks/utils.py b/server/dive_tasks/utils.py
index 9f2148952..8b443a78a 100644
--- a/server/dive_tasks/utils.py
+++ b/server/dive_tasks/utils.py
@@ -200,7 +200,7 @@ def upload_zipped_flat_media_files(
         gc.sendRestRequest("POST", f"/dive_rpc/postprocess/{str(root_folderId)}")
     else:
         manager.write(f"Message: {validation['message']}\n")
-        manager.updateStatus(JobStatus.ERROR)
+        raise Exception("Could not Validate media Files")
 
 
 def upload_exported_zipped_dataset(
diff --git a/server/tests/integration/test_zip_uploads.py b/server/tests/integration/test_zip_uploads.py
index d0bbea91e..9f0933ce6 100644
--- a/server/tests/integration/test_zip_uploads.py
+++ b/server/tests/integration/test_zip_uploads.py
@@ -9,7 +9,7 @@
 
 @pytest.mark.integration
 @pytest.mark.parametrize("user", zipUser.values())
-@pytest.mark.run(order=4)
+@pytest.mark.run(order=2)
 def test_user_creation(admin_client: GirderClient, user: dict):
     try:
         admin_client.createUser(
@@ -26,7 +26,7 @@ def test_user_creation(admin_client: GirderClient, user: dict):
 
 @pytest.mark.integration
 @pytest.mark.parametrize("user", zipUser.values())
-@pytest.mark.run(order=4)
+@pytest.mark.run(order=3)
 def test_reset_integration_env(user: dict):
     client = GirderClient(apiUrl='http://localhost:8010/api/v1')
     client.authenticate(username=user['login'], password=user['password'])

From e90ce517d3ff0362e77aeca62a7b0d5e8b3c9225 Mon Sep 17 00:00:00 2001
From: Brandon Davis <brandon.davis@kitware.com>
Date: Mon, 3 Jan 2022 11:25:50 -0500
Subject: [PATCH 16/16] Arbitrary depth support for zip imports (#1099)

* Arbitrary depth support for imports

* Respond to comments
---
 server/dive_server/crud_rpc.py               |   3 +-
 server/dive_tasks/tasks.py                   |  97 +++++-----
 server/tests/integration/conftest.py         | 186 +++++++++----------
 server/tests/integration/test_zip_uploads.py |  72 +++----
 4 files changed, 179 insertions(+), 179 deletions(-)

diff --git a/server/dive_server/crud_rpc.py b/server/dive_server/crud_rpc.py
index 21d5f26e5..cdf6279b9 100644
--- a/server/dive_server/crud_rpc.py
+++ b/server/dive_server/crud_rpc.py
@@ -374,7 +374,6 @@ def postprocess(
         Conversion of CSV annotations into track JSON
     """
     job_is_private = user.get(constants.UserPrivateQueueEnabledMarker, False)
-    auxiliary = crud.get_or_create_auxiliary_folder(dsFolder, user)
     isClone = fromMeta(dsFolder, constants.ForeignMediaIdMarker, None) is not None
     # add default confidence filter threshold to folder metadata
     dsFolder['meta'][constants.ConfidenceFiltersMarker] = {'default': 0.1}
@@ -413,6 +412,7 @@ def postprocess(
             )
             newjob.job[constants.JOBCONST_PRIVATE_QUEUE] = job_is_private
             Job().save(newjob.job)
+            return dsFolder
 
         # transcode VIDEO if necessary
         videoItems = Folder().childItems(
@@ -470,6 +470,7 @@ def make_file_generator(item):
             allFiles = [make_file_generator(item) for item in ymlItems]
             crud.saveTracks(dsFolder, meva.load_kpf_as_tracks(allFiles), user)
             ymlItems.rewind()
+            auxiliary = crud.get_or_create_auxiliary_folder(dsFolder, user)
             for item in ymlItems:
                 Item().move(item, auxiliary)
 
diff --git a/server/dive_tasks/tasks.py b/server/dive_tasks/tasks.py
index 514303c25..0762ebedb 100644
--- a/server/dive_tasks/tasks.py
+++ b/server/dive_tasks/tasks.py
@@ -125,7 +125,7 @@ def upgrade_pipelines(
     addons_to_update_update: List[Path] = []
 
     for addon in urls:
-        download_name = urlparse(addon).path.replace('/', '_')
+        download_name = urlparse(addon).path.replace(os.path.sep, '_')
         zipfile_path = conf.addon_zip_path / f'{download_name}.zip'
         if not zipfile_path.exists() or force:
             # Update the zipfile if force option set or file not exists
@@ -560,6 +560,12 @@ def convert_images(self: Task, folderId):
 
 @app.task(bind=True, acks_late=True, ignore_result=True)
 def extract_zip(self: Task, folderId: str, itemId: str):
+    """
+    Discovery logic:
+    * Find all folders that have at least one child file (potential datasets)
+    * Exclude folders which are sub-folders of previously discovered folders
+      because datasets cannot be nested in other datasets
+    """
     context: dict = {}
     gc: GirderClient = self.girder_client
     manager: JobManager = patch_manager(self.job_manager)
@@ -573,7 +579,7 @@ def extract_zip(self: Task, folderId: str, itemId: str):
         file_name = str(_working_directory_path / item['name'])
         manager.write(f'Fetching input from {itemId} to {file_name}...\n')
         gc.downloadItem(itemId, _working_directory, item["name"])
-        top_level_folders = {}
+        discovered_folders = {}
         with zipfile.ZipFile(file_name, 'r') as zipObj:
             listOfFileNames = zipObj.namelist()
             sum_file_size = sum([data.file_size for data in zipObj.filelist])
@@ -585,19 +591,25 @@ def extract_zip(self: Task, folderId: str, itemId: str):
                     Please contact an admin at viame-web@kitware.com if this is a valid zip file"
                 )
                 raise Exception("High Compression Ratio for Zip File")
+
             for fileName in listOfFileNames:
-                # contains a sub folder
-                if fileName.endswith('/') and fileName.count('/') == 1:
-                    top_level_folders[fileName.replace('/', '')] = 'flat'
-                # sub folder has a meta.json so it is imported
-                if fileName.endswith(constants.MetaFileName):
-                    root_folder = fileName.replace(f'/{constants.MetaFileName}', '')
-                    top_level_folders[root_folder] = 'dataset'
+                folderName = os.path.dirname(fileName)
+                parentName = os.path.dirname(folderName)
+                if parentName in discovered_folders and folderName != '':
+                    discovered_folders[folderName] = 'ignored'
+                    continue
+                if fileName.endswith(os.path.sep):
+                    continue
+                if folderName not in discovered_folders:
+                    discovered_folders[folderName] = 'unstructured'
+                if os.path.basename(fileName) == constants.MetaFileName:
+                    # sub folder has a meta.json so it is an exported dataset
+                    discovered_folders[folderName] = 'dataset'
                 if fileName.endswith('.zip'):
-                    manager.write("Nested Zip files are invalid\n")
                     raise Exception("Nested Zip Files are invalid")
                 manager.write(f"Extracting: {fileName}\n")
                 zipObj.extract(fileName, f'{_working_directory}')
+
         # remove the zip file so it isn't uploaded back to the folder
         os.remove(file_name)
         # Create source folder and move zip file there
@@ -610,47 +622,34 @@ def extract_zip(self: Task, folderId: str, itemId: str):
             "PUT",
             f"/item/{str(item['_id'])}?folderId={str(created_folder['_id'])}",
         )
+        # Only make subfolders if more than 1 discovered folder exists
+        make_subfolders = (
+            len(discovered_folders) - list(discovered_folders.values()).count('ignored')
+        ) > 1
+        for folderName, folderType in discovered_folders.items():
+            subFolderName = folderName if make_subfolders else ''
+            if folderType == 'unstructured':
+                utils.upload_zipped_flat_media_files(
+                    gc,
+                    manager,
+                    folderId,
+                    _working_directory_path / folderName,
+                    subFolderName,
+                )
+            elif folderType == 'dataset':
+                utils.upload_exported_zipped_dataset(
+                    gc,
+                    manager,
+                    folderId,
+                    _working_directory_path / folderName,
+                    subFolderName,
+                )
+            else:
+                manager.write(f'Ignoring {folderName}\n')
 
-        if len(top_level_folders) > 0:
-            for top_folder in top_level_folders.keys():
-                base_folder = ''  # default for no subfolder
-                # single flat/dataset import
-                if len(top_level_folders) != 1:
-                    base_folder = top_folder
-                # multiple flat/dataset import
-                if top_level_folders[top_folder] == 'flat':
-                    utils.upload_zipped_flat_media_files(
-                        gc,
-                        manager,
-                        folderId,
-                        Path(f"{_working_directory}/{top_folder}"),
-                        base_folder,
-                    )
-                # multiple flat file exports combined
-                elif top_level_folders[top_folder] == 'dataset':
-                    utils.upload_exported_zipped_dataset(
-                        gc,
-                        manager,
-                        folderId,
-                        Path(f"{_working_directory}/{top_folder}"),
-                        base_folder,
-                    )
-
-        else:  # flat zip file with media and/or annotation data inside
-            utils.upload_zipped_flat_media_files(gc, manager, folderId, Path(_working_directory))
-
-        # remove auxilary folder and metadata if there are sub folders
-        if len(top_level_folders) > 1:
-            sub_folders = list(gc.listFolder(folderId))
-            aux_folders = list(
-                (x for x in sub_folders if x['name'] == constants.AuxiliaryFolderName)
-            )
-            if len(aux_folders) > 0:
-                gc.delete(f"folder/{aux_folders[0]['_id']}")
-            # remove metadata
-            metdata = [constants.TypeMarker, constants.FPSMarker, constants.DatasetMarker]
+        if make_subfolders:
             gc.sendRestRequest(
                 "DELETE",
                 f"folder/{folderId}/metadata",
-                json=metdata,
+                json=[constants.TypeMarker, constants.FPSMarker, constants.DatasetMarker],
             )
diff --git a/server/tests/integration/conftest.py b/server/tests/integration/conftest.py
index fd30c39a0..b2af12c70 100644
--- a/server/tests/integration/conftest.py
+++ b/server/tests/integration/conftest.py
@@ -14,7 +14,7 @@
 Most tests run operations on a single dataset for each user, so keeping
 each user constrained to a single data type will help ensure robustness
 """
-users: Dict[str, Dict[str, Any]] = {
+users = {
     'alice': {
         'login': 'alice',
         'email': 'alice@locahost.lan',
@@ -147,99 +147,97 @@
     },
 }
 
-zipUser: Dict[str, Dict[str, Any]] = {
-    "zipUser": {
-        'login': 'zipUser',
-        'email': 'zipUser@locahost.lan',
-        'firstName': 'zip',
-        'lastName': 'User',
-        'password': 'zipUserPass',
-        'data': [
-            {
-                'name': 'testImageZip',
-                'path': 'zipTestFiles/testImageZip.zip',
-                'fps': 1,
-                'type': 'image-sequence',
-                'trackCount': 197,
-                'job_status': JobStatus.SUCCESS,
-            },
-            {
-                'name': 'testVideoZip',
-                'path': 'zipTestFiles/testVideoZip.zip',
-                'fps': 29.97002997002997,
-                'originalFps': 30000 / 1001,
-                'type': 'video',
-                'job_status': JobStatus.SUCCESS,
-            },
-            {
-                'name': 'singleDatasetImport',
-                'path': 'zipTestFiles/singleDatasetImport.zip',
-                'fps': 29.97002997002997,
-                'originalFps': 30000 / 1001,
-                'type': 'video',
-                'job_status': JobStatus.SUCCESS,
-            },
-            {
-                'name': 'flatMultiImport',
-                'path': 'zipTestFiles/flatMultiImport.zip',
-                'fps': -1,
-                'type': 'zip',
-                'subDatasets': [
-                    {
-                        'name': 'testVideoZip',
-                        'type': 'video',
-                        'fps': 29.97002997002997,
-                    },
-                    {
-                        'name': 'testImageZip',
-                        'type': 'image-sequence',
-                        'fps': 1,
-                    },
-                ],
-                'job_status': JobStatus.SUCCESS,
-            },
-            {
-                'name': 'MultiDatasetImport',
-                'path': 'zipTestFiles/MultiDatasetImport.zip',
-                'fps': -1,
-                'type': 'zip',
-                'subDatasets': [
-                    {
-                        'name': 'video1_train_mp4',
-                        'type': 'video',
-                        'fps': 29.97002997002997,
-                    },
-                    {
-                        'name': 'testtrain1_imagelist',
-                        'type': 'image-sequence',
-                        'fps': 1,
-                    },
-                ],
-                'job_status': JobStatus.SUCCESS,
-            },
-            {
-                'name': 'badFormatZip',
-                'path': 'zipTestFiles/badFormatZip.zip',
-                'fps': 1,
-                'type': 'image-sequence',
-                'job_status': JobStatus.ERROR,
-            },
-            {
-                'name': 'nestedZip',
-                'path': 'zipTestFiles/nestedZip.zip',
-                'fps': 1,
-                'type': 'image-sequence',
-                'job_status': JobStatus.ERROR,
-            },
-            {
-                'name': 'zipBomb',
-                'path': 'zipTestFiles/zipBomb.zip',
-                'fps': 1,
-                'type': 'image-sequence',
-                'job_status': JobStatus.ERROR,
-            },
-        ],
-    }
+zipUser = {
+    'login': 'zipUser',
+    'email': 'zipUser@locahost.lan',
+    'firstName': 'zip',
+    'lastName': 'User',
+    'password': 'zipUserPass',
+    'data': [
+        {
+            'name': 'testImageZip',
+            'path': 'zipTestFiles/testImageZip.zip',
+            'fps': 1,
+            'type': 'image-sequence',
+            'trackCount': 197,
+            'job_status': JobStatus.SUCCESS,
+        },
+        {
+            'name': 'testVideoZip',
+            'path': 'zipTestFiles/testVideoZip.zip',
+            'fps': 29.97002997002997,
+            'originalFps': 30000 / 1001,
+            'type': 'video',
+            'job_status': JobStatus.SUCCESS,
+        },
+        {
+            'name': 'SingleDatasetImport',
+            'path': 'zipTestFiles/SingleDatasetImport.zip',
+            'fps': 29.97002997002997,
+            'originalFps': 30000 / 1001,
+            'type': 'video',
+            'job_status': JobStatus.SUCCESS,
+        },
+        {
+            'name': 'flatMultiImport',
+            'path': 'zipTestFiles/flatMultiImport.zip',
+            'fps': -1,
+            'type': 'zip',
+            'subDatasets': [
+                {
+                    'name': 'testVideoZip',
+                    'type': 'video',
+                    'fps': 29.97002997002997,
+                },
+                {
+                    'name': 'testImageZip',
+                    'type': 'image-sequence',
+                    'fps': 1,
+                },
+            ],
+            'job_status': JobStatus.SUCCESS,
+        },
+        {
+            'name': 'MultiDatasetImport',
+            'path': 'zipTestFiles/MultiDatasetImport.zip',
+            'fps': -1,
+            'type': 'zip',
+            'subDatasets': [
+                {
+                    'name': 'video1_train_mp4',
+                    'type': 'video',
+                    'fps': 29.97002997002997,
+                },
+                {
+                    'name': 'testtrain1_imagelist',
+                    'type': 'image-sequence',
+                    'fps': 1,
+                },
+            ],
+            'job_status': JobStatus.SUCCESS,
+        },
+        {
+            'name': 'badFormatZip',
+            'path': 'zipTestFiles/badFormatZip.zip',
+            'fps': 1,
+            'type': 'image-sequence',
+            'job_status': JobStatus.ERROR,
+        },
+        {
+            'name': 'nestedZip',
+            'path': 'zipTestFiles/nestedZip.zip',
+            'fps': 1,
+            'type': 'image-sequence',
+            'job_status': JobStatus.ERROR,
+        },
+        {
+            'name': 'zipBomb',
+            'path': 'zipTestFiles/zipBomb.zip',
+            'fps': 1,
+            'type': 'image-sequence',
+            'job_status': JobStatus.ERROR,
+        },
+    ],
 }
 
 
diff --git a/server/tests/integration/test_zip_uploads.py b/server/tests/integration/test_zip_uploads.py
index 9f0933ce6..5d513fc15 100644
--- a/server/tests/integration/test_zip_uploads.py
+++ b/server/tests/integration/test_zip_uploads.py
@@ -8,7 +8,7 @@
 
 
 @pytest.mark.integration
-@pytest.mark.parametrize("user", zipUser.values())
+@pytest.mark.parametrize("user", [zipUser])
 @pytest.mark.run(order=2)
 def test_user_creation(admin_client: GirderClient, user: dict):
     try:
@@ -25,7 +25,7 @@ def test_user_creation(admin_client: GirderClient, user: dict):
 
 
 @pytest.mark.integration
-@pytest.mark.parametrize("user", zipUser.values())
+@pytest.mark.parametrize("user", [zipUser])
 @pytest.mark.run(order=3)
 def test_reset_integration_env(user: dict):
     client = GirderClient(apiUrl='http://localhost:8010/api/v1')
@@ -35,40 +35,42 @@ def test_reset_integration_env(user: dict):
 
 
 @pytest.mark.integration
-@pytest.mark.parametrize("user", zipUser.values())
+@pytest.mark.parametrize("dataset", zipUser['data'])
 @pytest.mark.run(order=4)
-def test_upload_zip_data(user: dict):
+def test_upload_zip_data(dataset: dict):
+    user = zipUser
     client = GirderClient(apiUrl='http://localhost:8010/api/v1')
     client.authenticate(username=user['login'], password=user['password'])
 
-    createdDatasets = []
-    for dataset in user['data']:
-        dsPath = localDataRoot / str(dataset['path'])
-        privateFolder = getTestFolder(client)
-        newDatasetFolder = client.createFolder(
-            privateFolder['_id'],
-            dataset['name'],
-            metadata={
-                'fps': dataset['fps'],
-                'type': dataset['type'],
-            },
-        )
-        createdDatasets.append(newDatasetFolder)
-        if Path(dsPath).is_file():
-            client.uploadFileToFolder(newDatasetFolder['_id'], str(dsPath))
-        client.post(f'dive_rpc/postprocess/{newDatasetFolder["_id"]}')
-        try:
-            wait_for_jobs(client, max_wait_timeout=30, expected_status=dataset['job_status'])
-        except Exception as ex:
-            if dataset['job_status'] == JobStatus.ERROR:
-                continue
-            raise ex
-        # verify sub datasets if they exist
-        if dataset.get('subDatasets', False):
-            folders = list(client.listFolder(newDatasetFolder['_id']))
-            for item in dataset["subDatasets"]:
-                matches = [x for x in folders if x["name"] == item["name"]]
-                if len(matches) > 0:
-                    meta = matches[0].get("meta", {})
-                    assert meta.get("fps", -1) == item["fps"]
-                    assert meta.get("type", "") == item["type"]
+    dsPath = localDataRoot / str(dataset['path'])
+    privateFolder = getTestFolder(client)
+    newDatasetFolder = client.createFolder(
+        privateFolder['_id'],
+        dataset['name'],
+        metadata={
+            'fps': dataset['fps'],
+            'type': dataset['type'],
+        },
+    )
+    if Path(dsPath).is_file():
+        client.uploadFileToFolder(newDatasetFolder['_id'], str(dsPath))
+    client.post(f'dive_rpc/postprocess/{newDatasetFolder["_id"]}')
+    wait_for_jobs(client, max_wait_timeout=30, expected_status=dataset['job_status'])
+
+    resultFolder = client.getFolder(newDatasetFolder['_id'])
+    # verify sub datasets if they exist
+    if dataset.get('subDatasets', False):
+        folders = list(client.listFolder(newDatasetFolder['_id']))
+        for item in dataset["subDatasets"]:
+            matches = [x for x in folders if x["name"] == item["name"]]
+            if len(matches) > 0:
+                meta = matches[0].get("meta", {})
+                assert meta.get("fps", -1) == item["fps"]
+                assert meta.get("type", "") == item["type"]
+                assert meta.get("annotate", False)
+    elif dataset['job_status'] == JobStatus.SUCCESS:
+        assert resultFolder['meta'].get("annotate", False)
+        assert type(resultFolder['meta'].get("fps")) in [int, float]
+        assert type(resultFolder['meta'].get("type")) == str
+    else:
+        assert resultFolder['meta'].get("annotate", None) is None