diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index 66a8079c8..363842ed7 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -89,6 +89,7 @@ interface DatasetMetaMutable { confidenceFilters?: Record; attributes?: Readonly>; } +const DatasetMetaMutableKeys = ['attributes', 'confidenceFilters', 'customTypeStyles']; interface DatasetMeta extends DatasetMetaMutable { id: Readonly; @@ -144,10 +145,11 @@ export { useApi, }; -export type { +export { Api, DatasetMeta, DatasetMetaMutable, + DatasetMetaMutableKeys, DatasetType, SubType, FrameImage, diff --git a/client/platform/desktop/backend/ipcService.ts b/client/platform/desktop/backend/ipcService.ts index 35316e91d..b145af186 100644 --- a/client/platform/desktop/backend/ipcService.ts +++ b/client/platform/desktop/backend/ipcService.ts @@ -94,9 +94,7 @@ export default function register() { }); ipcMain.handle('import-annotation', async (event, { id, path }: { id: string; path: string }) => { - const ret = await common.annotationImport( - settings.get(), id, path, - ); + const ret = await common.dataFileImport(settings.get(), id, path); return ret; }); diff --git a/client/platform/desktop/backend/native/common.spec.ts b/client/platform/desktop/backend/native/common.spec.ts index df80788f5..01c395274 100644 --- a/client/platform/desktop/backend/native/common.spec.ts +++ b/client/platform/desktop/backend/native/common.spec.ts @@ -105,6 +105,8 @@ const convertMedia = async (settingsVal: Settings, args: ConversionArgs, // eslint-disable-next-line @typescript-eslint/no-unused-vars const console = new Console(process.stdout, process.stderr); +const emptyCsvString = '# comment line\n# metadata,fps: 32,"whatever"\n#comment line'; + mockfs({ '/opt/viame': { configs: { @@ -116,6 +118,11 @@ mockfs({ }, }, '/home/user/data': { + annotationImport: { + 'viame.csv': emptyCsvString, + 'foreign.meta.json': '{ "confidenceFilters": {"default": 0.8}, "type": "invalidtype" }', + 'dive.json': '{ "0": { "trackId": 0 } }', // fake track file + }, imageSuccess: { 'foo.png': '', 'bar.png': '', @@ -165,7 +172,7 @@ mockfs({ imageSuccessWithAnnotations: { 'foo.png': '', 'bar.png': '', - 'file1.csv': '# comment line\n# metadata,fps: 32,"whateever"\n#comment line', + 'file1.csv': emptyCsvString, }, videoSuccess: { 'video1.avi': '', @@ -488,6 +495,30 @@ describe('native.common', () => { )).rejects.toThrowError('Found non-image type data in image list file'); }); + it('dataFileImport', async () => { + const payload = await common.beginMediaImport( + settings, '/home/user/data/imageLists/success/image_list.txt', checkMedia, + ); + const final = await common.finalizeMediaImport(settings, payload, updater, convertMedia); + const tracks = await common.loadDetections(settings, final.id); + expect(Object.keys(tracks)).toHaveLength(0); + + await common.dataFileImport(settings, final.id, '/home/user/data/annotationImport/dive.json'); + const tracks1 = await common.loadDetections(settings, final.id); + expect(Object.keys(tracks1)).toHaveLength(1); + + await common.dataFileImport(settings, final.id, '/home/user/data/annotationImport/viame.csv'); + const tracks2 = await common.loadDetections(settings, final.id); + expect(Object.keys(tracks2)).toHaveLength(0); + const meta = await common.loadMetadata(settings, final.id, urlMapper); + expect(meta.fps).toBe(32); + + await common.dataFileImport(settings, final.id, '/home/user/data/annotationImport/foreign.meta.json'); + const meta2 = await common.loadMetadata(settings, final.id, urlMapper); + expect(meta2.confidenceFilters).toStrictEqual({ "default": 0.8 }); + expect(meta2.type).toBe("image-sequence"); // Ensure meta import cannot change immutable fields. + }); + it('import with CSV annotations without specifying track file', async () => { const payload = await common.beginMediaImport(settings, '/home/user/data/imageSuccessWithAnnotations', checkMedia); payload.trackFileAbsPath = ''; //It returns null be default but users change it. diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index 720ae8eec..6bbf07156 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -8,17 +8,22 @@ import { shell } from 'electron'; import mime from 'mime-types'; import moment from 'moment'; import lockfile from 'proper-lockfile'; +import { + cloneDeep, merge, uniq, pick, +} from 'lodash'; import { DefaultConfidence } from 'vue-media-annotator/use/useTrackFilters'; +import { TrackData } from 'vue-media-annotator/track'; import { DatasetType, MultiTrackRecord, Pipelines, SaveDetectionsArgs, FrameImage, DatasetMetaMutable, TrainingConfigs, SaveAttributeArgs, MultiCamMedia, + DatasetMetaMutableKeys, } from 'dive-common/apispec'; import * as viameSerializers from 'platform/desktop/backend/serializers/viame'; import * as nistSerializers from 'platform/desktop/backend/serializers/nist'; import { - websafeImageTypes, websafeVideoTypes, otherImageTypes, otherVideoTypes, MultiType, VideoType, + websafeImageTypes, websafeVideoTypes, otherImageTypes, otherVideoTypes, MultiType, } from 'dive-common/constants'; import { JsonMeta, Settings, JsonMetaCurrentVersion, DesktopMetadata, DesktopJobUpdater, @@ -27,12 +32,12 @@ import { import { cleanString, filterByGlob, makeid, strNumericCompare, } from 'platform/desktop/sharedUtils'; -import { Attribute, Attributes } from 'vue-media-annotator/use/useAttributes'; -import { cloneDeep, uniq } from 'lodash'; + import processTrackAttributes from './attributeProcessor'; import { upgrade } from './migrations'; import { getMultiCamUrls, transcodeMultiCam } from './multiCamUtils'; + const ProjectsFolderName = 'DIVE_Projects'; const JobsFolderName = 'DIVE_Jobs'; const PipelinesFolderName = 'DIVE_Pipelines'; @@ -52,6 +57,21 @@ async function readLines(filePath: string): Promise { return rawBuffer.toString().replace(/\r\n/g, '\n').split('\n'); } +/** + * Read a text file as json + */ +async function _loadAsJson(abspath: string) { + const rawBuffer = await fs.readFile(abspath, 'utf-8'); + if (rawBuffer.length === 0) { + return false; + } + try { + return JSON.parse(rawBuffer); + } catch (err) { + throw new Error(`Unable to parse ${abspath}: ${err}`); + } +} + /** * findImagesInFolder * Import either a directory of images or images from a text file @@ -205,13 +225,7 @@ async function getValidatedProjectDir(settings: Settings, datasetId: string) { * @param metaPath a known, existing path */ async function loadJsonMetadata(metaAbsPath: string): Promise { - const rawBuffer = await fs.readFile(metaAbsPath, 'utf-8'); - let metaJson; - try { - metaJson = JSON.parse(rawBuffer); - } catch (err) { - throw new Error(`Unable to parse ${metaAbsPath}: ${err}`); - } + const metaJson = await _loadAsJson(metaAbsPath); /* check if this file meets the current schema version */ upgrade(metaJson); return metaJson as JsonMeta; @@ -222,16 +236,7 @@ async function loadJsonMetadata(metaAbsPath: string): Promise { * @param tracksPath a known, existing path */ async function loadJsonTracks(tracksAbsPath: string): Promise { - const rawBuffer = await fs.readFile(tracksAbsPath, 'utf-8'); - if (rawBuffer.length === 0) { - return {}; // Return empty object if file was empty - } - let annotationData: MultiTrackRecord = {}; - try { - annotationData = JSON.parse(rawBuffer) as MultiTrackRecord; - } catch (err) { - throw new Error(`Unable to parse ${tracksAbsPath}: ${err}`); - } + const annotationData = await _loadAsJson(tracksAbsPath); // TODO: somehow verify the schema of this file if (Array.isArray(annotationData)) { throw new Error('object expected in track json'); @@ -505,8 +510,7 @@ async function _saveAsJson(absPath: string, data: unknown) { await fs.writeFile(absPath, serialized); } -async function saveMetadata(settings: Settings, datasetId: string, - args: DatasetMetaMutable & { attributes?: Record }) { +async function saveMetadata(settings: Settings, datasetId: string, args: DatasetMetaMutable) { const projectDirInfo = await getValidatedProjectDir(settings, datasetId); const release = await _acquireLock(projectDirInfo.basePath, projectDirInfo.metaFileAbsPath, 'meta'); const existing = await loadJsonMetadata(projectDirInfo.metaFileAbsPath); @@ -545,41 +549,53 @@ async function saveAttributes(settings: Settings, datasetId: string, args: SaveA } -async function processAnnotationFilePath( +async function _ingestFilePath( settings: Settings, datasetId: string, path: string, -) { +): Promise<(DatasetMetaMutable & { fps?: number }) | null> { if (!fs.existsSync(path)) { - return {}; + return null; + } + if (fs.statSync(path).size === 0) { + return null; } - if (fs.statSync(path).size > 0) { - // Attempt to process the file - let data: viameSerializers.AnnotationFileData; - if (npath.extname(path) === '.json' && await nistSerializers.confirmNistFile(path)) { - data = await nistSerializers.loadNistFile(path); + // Make a copy of the file in aux + const projectInfo = getProjectDir(settings, datasetId); + const newPath = npath.join(projectInfo.auxDirAbsPath, `imported_${npath.basename(path)}`); + await fs.copy(path, newPath); + // Attempt to process the file + let tracks: TrackData[] | null = null; + const meta: DatasetMetaMutable & { fps?: number } = {}; + if (JsonFileName.test(path)) { + const jsonObject = await _loadAsJson(path); + if (nistSerializers.confirmNistFormat(jsonObject)) { + // NIST json file + const data = await nistSerializers.loadNistFile(path); + tracks = data.tracks; + meta.fps = data.fps; + } else if (DatasetMetaMutableKeys.some((key) => key in jsonObject)) { + // DIVE Json metadata config file + merge(meta, pick(jsonObject, DatasetMetaMutableKeys)); } else { - data = await viameSerializers.parseFile(path); - } - try { - // eslint-disable-next-line no-await-in-loop - const processed = processTrackAttributes(data.tracks); - const { fps } = data; - const { attributes } = processed; - // eslint-disable-next-line no-await-in-loop - await _saveSerialized(settings, datasetId, processed.data, true); - return { - fps, attributes, path, - }; - } catch (err) { - // eslint-disable-next-line no-continue + // Regular dive json + tracks = Object.values(await loadJsonTracks(path)); } - return {}; + } else if (CsvFileName.test(path)) { + // VIAME CSV File + const data = await viameSerializers.parseFile(path); + tracks = data.tracks; + meta.fps = data.fps; } - return { }; + if (tracks !== null) { + const processed = processTrackAttributes(tracks); + await _saveSerialized(settings, datasetId, processed.data, true); + } + return meta; } + /** - * processOtherAnnotationFiles imports data from external annotation formats + * ingestDataFiles imports data from external annotation formats * given a list of candidate file paths. * * SUPPORTED FORMATS: @@ -591,31 +607,28 @@ async function processAnnotationFilePath( * @param multiCamResults Objec where the keys are Camera names * and the value is the path to a result file */ -async function processOtherAnnotationFiles( +async function ingestDataFiles( settings: Settings, datasetId: string, absPaths: string[], multiCamResults?: Record, -): Promise<{ fps?: number; processedFiles: string[]; attributes?: Attributes }> { - let fps: number | undefined; +): Promise<{ + processedFiles: string[]; + meta: DatasetMetaMutable & { fps?: number }; +}> { const processedFiles = []; // which files were processed to generate the detections - let attributes: Attributes = {}; + const meta = {}; for (let i = 0; i < absPaths.length; i += 1) { const path = absPaths[i]; // eslint-disable-next-line no-await-in-loop - const result = await processAnnotationFilePath(settings, datasetId, path); - if (result.fps) { - fps = fps || result.fps; - } - if (result.attributes) { - attributes = result.attributes; - } - if (result.path) { - processedFiles.push(result.path); + const newMeta = await _ingestFilePath(settings, datasetId, path); + if (newMeta !== null) { + merge(meta, newMeta); + processedFiles.push(path); } } - //Processing of multiCam results: + // processing of multiCam results if (multiCamResults) { const cameraAndPath = Object.entries(multiCamResults); for (let i = 0; i < cameraAndPath.length; i += 1) { @@ -623,20 +636,15 @@ async function processOtherAnnotationFiles( const path = cameraAndPath[i][1]; const cameraDatasetId = `${datasetId}/${cameraName}`; // eslint-disable-next-line no-await-in-loop - const result = await processAnnotationFilePath(settings, cameraDatasetId, path); - if (result.fps) { - fps = fps || result.fps; - } - if (result.attributes) { - attributes = result.attributes; - } - if (result.path) { - processedFiles.push(result.path); + const newMeta = await _ingestFilePath(settings, cameraDatasetId, path); + if (newMeta !== null) { + merge(meta, newMeta); + processedFiles.push(path); } } } - return { fps, processedFiles, attributes }; + return { processedFiles, meta }; } /** * Need to take the trained pipeline if it exists and place it in the DIVE_Pipelines folder @@ -842,73 +850,13 @@ async function beginMediaImport( }; } -async function annotationImport( - settings: Settings, - id: string, - annotationPath: string, - allowEmpty = true, -) { - const projectInfo = getProjectDir(settings, id); - const validatedInfo = await getValidatedProjectDir(settings, id); - const jsonMeta = await loadJsonMetadata(projectInfo.metaFileAbsPath); - - // If it is a json file we need to make sure it has the proper extension - if (JsonFileName.test(npath.basename(annotationPath))) { - const statResult = await fs.stat(annotationPath); - if (statResult.isFile()) { - //Check so see if it is a NIST File before moving - const nistFormat = await nistSerializers.loadNistFile(annotationPath); - if (nistFormat && jsonMeta.type !== VideoType) { - throw new Error(`Dataset is of type: ${jsonMeta.type} not ${VideoType}. NIST formats can only be imported on ${VideoType} datasets`); - } - const release = await _acquireLock(projectInfo.basePath, projectInfo.basePath, 'tracks'); - try { - await fs.move( - validatedInfo.trackFileAbsPath, - npath.join( - validatedInfo.auxDirAbsPath, - npath.basename(validatedInfo.trackFileAbsPath), - ), - ); - } catch (err) { - // Some part of the project dir didn't exist - if (!allowEmpty) throw err; - } - const time = moment().format('MM-DD-YYYY_hh-mm-ss.SSS'); - const newFileName = `result_${time}.json`; - - const newPath = npath.join(projectInfo.basePath, npath.basename(newFileName)); - if (nistFormat) { - const trackData = await nistSerializers.loadNistFile(annotationPath); - const trackStructure: MultiTrackRecord = {}; - for (let i = 0; i < trackData.tracks.length; i += 1) { - const track = trackData.tracks[i]; - trackStructure[track.trackId] = track; - } - const serialized = JSON.stringify(trackStructure, null, 2); - await fs.writeFile(newPath, serialized); - } else { - await fs.copy( - annotationPath, - newPath, - ); - } - await release(); - return true; - } - throw new Error(`${annotationPath} is not a valid file`); - } - // If not a JSON we do a process for the CSV - const newPath = npath.join(projectInfo.basePath, npath.basename(annotationPath)); - await fs.copy( - annotationPath, - newPath, - ); - const results = await processOtherAnnotationFiles(settings, id, [newPath]); - if (results.processedFiles.length) { - return true; - } - return false; +async function dataFileImport(settings: Settings, id: string, path: string) { + const result = await ingestDataFiles(settings, id, [path]); + const projectDirData = await getValidatedProjectDir(settings, id); + const jsonMeta = await loadJsonMetadata(projectDirData.metaFileAbsPath); + merge(jsonMeta, result.meta); + await _saveAsJson(npath.join(projectDirData.basePath, JsonMetaFileName), jsonMeta); + return result; } async function _importTrackFile( @@ -918,52 +866,15 @@ async function _importTrackFile( jsonMeta: JsonMeta, userTrackFileAbsPath: string, ) { - /* Look for JSON track file as first priority */ - let foundDetections = false; - - if (userTrackFileAbsPath && !CsvFileName.test(userTrackFileAbsPath)) { - /* Move the track file into the new project directory */ - const time = moment().format('MM-DD-YYYY_hh-mm-ss.SSS'); - const newFileName = `result_${time}.json`; - - const newPath = npath.join(projectDirAbsPath, npath.basename(newFileName)); - - if (jsonMeta.type === VideoType - && await nistSerializers.confirmNistFile(userTrackFileAbsPath)) { - const trackData = await nistSerializers.loadNistFile(userTrackFileAbsPath); - const trackStructure: MultiTrackRecord = {}; - for (let i = 0; i < trackData.tracks.length; i += 1) { - const track = trackData.tracks[i]; - trackStructure[track.trackId] = track; - } - const serialized = JSON.stringify(trackStructure); - await fs.writeFile(newPath, serialized); - } else { - await fs.copy( - userTrackFileAbsPath, - newPath, - ); + if (userTrackFileAbsPath) { + const processed = await ingestDataFiles(settings, dsId, [userTrackFileAbsPath]); + merge(jsonMeta, processed.meta); + if (processed.processedFiles.length === 0) { + await _saveSerialized(settings, dsId, {}, true); } - //Load tracks to generate attributes - const tracks = await loadJsonTracks(newPath); - const { attributes } = processTrackAttributes(Object.values(tracks)); - // eslint-disable-next-line no-param-reassign - if (attributes) jsonMeta.attributes = attributes; - foundDetections = true; - } - if (!foundDetections && userTrackFileAbsPath && CsvFileName.test(userTrackFileAbsPath)) { - const csvFileCandidates = [userTrackFileAbsPath]; - - const { fps, processedFiles, attributes } = await processOtherAnnotationFiles( - settings, dsId, csvFileCandidates, - ); - // eslint-disable-next-line no-param-reassign - if (fps) jsonMeta.fps = fps; - // eslint-disable-next-line no-param-reassign - if (attributes) jsonMeta.attributes = attributes; - foundDetections = processedFiles.length > 0; + } else { + await _saveSerialized(settings, dsId, {}, true); } - /* custom image sort */ if (jsonMeta.imageListPath === undefined) { jsonMeta.originalImageFiles.sort(strNumericCompare); @@ -971,13 +882,7 @@ async function _importTrackFile( if (jsonMeta.transcodedImageFiles) { jsonMeta.transcodedImageFiles.sort(strNumericCompare); } - await _saveAsJson(npath.join(projectDirAbsPath, JsonMetaFileName), jsonMeta); - - /* create an empty file as fallback */ - if (!foundDetections) { - await _saveSerialized(settings, dsId, {}, true); - } return jsonMeta; } @@ -1071,7 +976,6 @@ async function finalizeMediaImport( jsonClone.transcodedVideoFile = cameraData.transcodedVideoFile || ''; jsonClone.transcodedImageFiles = cameraData.transcodedImageFiles || []; jsonClone.subType = null; - // eslint-disable-next-line no-await-in-loop const cameraDirAbsPath = await _initializeProjectDir(settings, jsonClone); let multiCamTrackFile = ''; @@ -1079,8 +983,9 @@ async function finalizeMediaImport( multiCamTrackFile = args.multiCamTrackFiles[cameraName]; } // eslint-disable-next-line no-await-in-loop - await _importTrackFile(settings, jsonClone.id, - cameraDirAbsPath, jsonClone, multiCamTrackFile); + await _importTrackFile( + settings, jsonClone.id, cameraDirAbsPath, jsonClone, multiCamTrackFile, + ); } } const finalJsonMeta = await _importTrackFile( @@ -1122,9 +1027,9 @@ export { JobsFolderName, autodiscoverData, beginMediaImport, + dataFileImport, deleteDataset, checkDataset, - annotationImport, createKwiverRunWorkingDir, exportDataset, finalizeMediaImport, @@ -1137,7 +1042,7 @@ export { loadJsonTracks, loadDetections, openLink, - processOtherAnnotationFiles, + ingestDataFiles, saveDetections, saveMetadata, completeConversion, diff --git a/client/platform/desktop/backend/native/viame.ts b/client/platform/desktop/backend/native/viame.ts index 6b9865d93..ac6fbc369 100644 --- a/client/platform/desktop/backend/native/viame.ts +++ b/client/platform/desktop/backend/native/viame.ts @@ -181,11 +181,11 @@ async function runPipeline( job.on('exit', async (code) => { if (code === 0) { try { - const { attributes } = await common.processOtherAnnotationFiles( + const { meta: newMeta } = await common.ingestDataFiles( settings, datasetId, [detectorOutput, trackOutput], multiOutFiles, ); - if (attributes) { - meta.attributes = attributes; + if (newMeta) { + meta.attributes = newMeta.attributes; await common.saveMetadata(settings, datasetId, meta); } } catch (err) { diff --git a/client/platform/desktop/backend/serializers/nist.ts b/client/platform/desktop/backend/serializers/nist.ts index e67b47118..38c346268 100644 --- a/client/platform/desktop/backend/serializers/nist.ts +++ b/client/platform/desktop/backend/serializers/nist.ts @@ -66,19 +66,6 @@ interface NistActivity { function confirmNistFormat(data: any): data is NistFile { return Array.isArray(data?.activities) && Array.isArray(data?.filesProcessed); } -async function confirmNistFile(filename: string) { - const rawBuffer = await fs.readFile(filename, 'utf-8'); - let nistJson; - if (rawBuffer.length === 0) { - return false; - } - try { - nistJson = JSON.parse(rawBuffer); - } catch (err) { - throw new Error(`Unable to parse ${filename}: ${err}`); - } - return confirmNistFormat(nistJson); -} function loadObjects( objects: NistObject[], @@ -392,5 +379,4 @@ export { convertNisttoJSON, exportNist, confirmNistFormat, - confirmNistFile, };