From 93936506f67c137bc518c5557e58527a2a9cd861 Mon Sep 17 00:00:00 2001 From: Brandon Davis Date: Wed, 16 Dec 2020 09:31:33 -0500 Subject: [PATCH 1/8] WIP --- .../desktop/backend/platforms/common.ts | 6 ++++- client/viame-web-common/apispec.ts | 23 +++++++++++++++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/client/platform/desktop/backend/platforms/common.ts b/client/platform/desktop/backend/platforms/common.ts index 20b53c8e4..7db7dafce 100644 --- a/client/platform/desktop/backend/platforms/common.ts +++ b/client/platform/desktop/backend/platforms/common.ts @@ -7,7 +7,7 @@ import { shell } from 'electron'; import mime from 'mime-types'; import moment from 'moment'; import { TrackData } from 'vue-media-annotator/track'; -import { DatasetType, Pipelines, SaveDetectionsArgs } from 'viame-web-common/apispec'; +import { DatasetType, Pipelines, DatasetSchema, SaveDetectionsArgs } from 'viame-web-common/apispec'; import { Settings, websafeImageTypes } from '../../constants'; import * as viameSerializers from '../serializers/viame'; @@ -261,6 +261,10 @@ async function postprocess(paths: string[], datasetId: string) { } } +async function loadDataset(): Promise { + +} + async function openLink(url: string) { shell.openExternal(url); } diff --git a/client/viame-web-common/apispec.ts b/client/viame-web-common/apispec.ts index 075e7f81e..20c8d97ef 100644 --- a/client/viame-web-common/apispec.ts +++ b/client/viame-web-common/apispec.ts @@ -49,14 +49,32 @@ interface DatasetMetaMutable { confidenceFilters?: Record; } -interface DatasetMeta extends DatasetMetaMutable { +export interface DatasetMeta extends DatasetMetaMutable { type: Readonly; fps: Readonly; imageData: FrameImage[]; videoUrl: string | undefined; } + +/** + * DatasetSchema is a structure that describes everything about + * media that could be opened in DIVE. This schema is JSON + * serializable (no maps, sets, or classes in the tree) + */ +export interface DatasetSchema { + version: number; + // TODO: in a future version attributes will be part of the dataset schema + // attributes: Attribute[]; + meta: DatasetMeta; + tracks: { [key: string]: TrackData }; +} + interface Api { + loadDataset(): Promise; + /** + * @deprecated soon attributes will come from loadDataset() + */ getAttributes(): Promise; setAttribute({ addNew, data }: {addNew: boolean | undefined; data: Attribute}): Promise; deleteAttribute(data: Attribute): Promise; @@ -67,10 +85,7 @@ interface Api { getTrainingConfigurations(): Promise; runTraining(folderId: string, pipelineName: string, config: string): Promise; - loadDetections(datasetId: string): Promise<{ [key: string]: TrackData }>; saveDetections(datasetId: string, args: SaveDetectionsArgs): Promise; - - loadMetadata(datasetId: string): Promise; saveMetadata(datasetId: string, metadata: DatasetMetaMutable): Promise; } From a54a4c31b187caba394261feaef622790512c2a1 Mon Sep 17 00:00:00 2001 From: Brandon Davis Date: Mon, 21 Dec 2020 09:07:34 -0500 Subject: [PATCH 2/8] WI{ --- client/platform/desktop/api/main.ts | 78 +----- client/platform/desktop/backend/ipcService.ts | 33 ++- .../desktop/backend/platforms/common.ts | 261 +++++++++++++----- .../desktop/backend/platforms/linux.ts | 3 +- .../desktop/backend/platforms/utils.ts | 24 ++ .../desktop/backend/platforms/windows.ts | 3 +- client/platform/desktop/backend/server.ts | 15 + client/platform/desktop/background.ts | 3 +- client/platform/desktop/constants.ts | 40 ++- client/platform/desktop/store/index.ts | 5 +- client/platform/desktop/store/settings.ts | 3 + client/viame-web-common/apispec.ts | 28 +- 12 files changed, 318 insertions(+), 178 deletions(-) create mode 100644 client/platform/desktop/backend/platforms/utils.ts diff --git a/client/platform/desktop/api/main.ts b/client/platform/desktop/api/main.ts index ccaabf3e1..dfa827b45 100644 --- a/client/platform/desktop/api/main.ts +++ b/client/platform/desktop/api/main.ts @@ -1,30 +1,23 @@ -import { AddressInfo } from 'net'; -import path from 'path'; +import type { FileFilter } from 'electron'; -import { ipcRenderer, remote, FileFilter } from 'electron'; -import fs from 'fs-extra'; -import mime from 'mime-types'; +import { ipcRenderer, remote } from 'electron'; import { Attribute, DatasetMetaMutable, - DatasetType, FrameImage, - Pipe, - Pipelines, TrainingConfigs, + DatasetType, Pipelines, TrainingConfigs, } from 'viame-web-common/apispec'; import common from '../backend/platforms/common'; import { DesktopJob, NvidiaSmiReply, RunPipeline, - websafeImageTypes, websafeVideoTypes, - DesktopDataset, Settings, + websafeVideoTypes, Settings, } from '../constants'; + const { loadDetections, saveDetections } = common; -function mediaServerInfo(): Promise { - return ipcRenderer.invoke('info'); -} + function nvidiaSmi(): Promise { return ipcRenderer.invoke('nvidia-smi'); @@ -78,68 +71,14 @@ async function runTraining( return Promise.resolve(); } -async function loadMetadata(datasetId: string): Promise { - let datasetType = undefined as 'video' | 'image-sequence' | undefined; - let videoUrl = ''; - let videoPath = ''; - let basePath = datasetId; // default to image-sequence type basepath - const imageData = [] as FrameImage[]; - const serverInfo = await mediaServerInfo(); - - function processFile(abspath: string) { - const basename = path.basename(abspath); - const abspathuri = `http://localhost:${serverInfo.port}/api/media?path=${abspath}`; - const mimetype = mime.lookup(abspath); - if (mimetype && websafeVideoTypes.includes(mimetype)) { - datasetType = 'video'; - basePath = path.dirname(datasetId); // parent directory of video; - videoPath = abspath; - videoUrl = abspathuri; - } else if (mimetype && websafeImageTypes.includes(mimetype)) { - datasetType = 'image-sequence'; - imageData.push({ - url: abspathuri, - filename: basename, - }); - } - } - - const info = await fs.stat(datasetId); - - if (info.isDirectory()) { - const contents = await fs.readdir(datasetId); - for (let i = 0; i < contents.length; i += 1) { - processFile(path.join(datasetId, contents[i])); - } - } else { - processFile(datasetId); - } - - if (datasetType === undefined) { - throw new Error(`Cannot open dataset ${datasetId}: No images or video found`); - } - - return Promise.resolve({ - name: path.basename(datasetId), - basePath, - videoPath, - meta: { - type: datasetType, - fps: 10, - imageData: datasetType === 'image-sequence' ? imageData : [], - videoUrl: datasetType === 'video' ? videoUrl : undefined, - }, - }); -} - // eslint-disable-next-line async function saveMetadata(datasetId: string, metadata: DatasetMetaMutable) { return Promise.resolve(); } -async function runPipeline(itemId: string, pipeline: Pipe, settings: Settings) { +async function runPipeline(itemId: string, pipeline: string, settings: Settings) { const args: RunPipeline = { - pipelineName: pipeline.name, + pipelineName: pipeline, datasetId: itemId, settings, }; @@ -158,7 +97,6 @@ export { runTraining, loadDetections, saveDetections, - loadMetadata, saveMetadata, /* Nonstandard APIs */ openFromDisk, diff --git a/client/platform/desktop/backend/ipcService.ts b/client/platform/desktop/backend/ipcService.ts index e5d1fa9b2..d2d6f90b1 100644 --- a/client/platform/desktop/backend/ipcService.ts +++ b/client/platform/desktop/backend/ipcService.ts @@ -3,28 +3,34 @@ import OS from 'os'; import { ipcMain } from 'electron'; import { DesktopJobUpdate, RunPipeline, Settings } from '../constants'; -import server from './server'; import linux from './platforms/linux'; import win32 from './platforms/windows'; import common from './platforms/common'; -export default function register() { - ipcMain.handle('info', () => { - const addr = server.address(); - return addr; - }); +let settings: Settings; +function getSetting() { + if (settings === undefined) { + throw new Error('Settings has not been initialized!'); + } + return settings; +} + +export default function register() { /** * Platform-agnostic methods */ - ipcMain.handle('get-pipeline-list', async (_, settings: Settings) => { - const ret = await common.getPipelineList(settings); + ipcMain.handle('get-pipeline-list', async () => { + const ret = await common.getPipelineList(getSetting()); return ret; }); ipcMain.handle('open-link-in-browser', (_, url: string) => { common.openLink(url); }); + ipcMain.on('update-settings', async (_, s: Settings) => { + settings = s; + }); /** * Platform-dependent methods @@ -32,6 +38,7 @@ export default function register() { // defaults to linux if win32 doesn't exist const currentPlatform = OS.platform() === 'win32' ? win32 : linux; + if (OS.platform() === 'win32') { win32.initialize(); } @@ -39,19 +46,21 @@ export default function register() { const ret = await currentPlatform.nvidiaSmi(); return ret; }); - ipcMain.handle('default-settings', async () => { const defaults = currentPlatform.DefaultSettings; return defaults; }); - ipcMain.handle('validate-settings', async (_, settings: Settings) => { - const ret = await currentPlatform.validateViamePath(settings); + ipcMain.handle('validate-settings', async () => { + const ret = await currentPlatform.validateViamePath(getSetting()); return ret; + }); + ipcMain.handle('import-media', async () => { + }); ipcMain.handle('run-pipeline', async (event, args: RunPipeline) => { const updater = (update: DesktopJobUpdate) => { event.sender.send('job-update', update); }; - return currentPlatform.runPipeline(args, updater); + return currentPlatform.runPipeline(getSetting(), args, updater); }); } diff --git a/client/platform/desktop/backend/platforms/common.ts b/client/platform/desktop/backend/platforms/common.ts index 7db7dafce..e0746ba18 100644 --- a/client/platform/desktop/backend/platforms/common.ts +++ b/client/platform/desktop/backend/platforms/common.ts @@ -7,103 +7,151 @@ import { shell } from 'electron'; import mime from 'mime-types'; import moment from 'moment'; import { TrackData } from 'vue-media-annotator/track'; -import { DatasetType, Pipelines, DatasetSchema, SaveDetectionsArgs } from 'viame-web-common/apispec'; +import { + DatasetType, Pipelines, DatasetSchema, SaveDetectionsArgs, FrameImage, +} from 'viame-web-common/apispec'; -import { Settings, websafeImageTypes } from '../../constants'; -import * as viameSerializers from '../serializers/viame'; +import { + JsonMeta, Settings, websafeImageTypes, websafeVideoTypes, JsonMetaCurrentVersion, DesktopDataset, +} from 'platform/desktop/constants'; +import * as viameSerializers from 'platform/desktop/backend/serializers/viame'; +import { cleanString, makeid } from './utils'; + +const ProjectsFolderName = 'DIVE_Projects'; +const TrainedPipelinesFolderName = 'DIVE_Trained_Pipelines'; const AuxFolderName = 'auxiliary'; const JobFolderName = 'job_runs'; -// Match examples: -// result_09-14-2020_14-49-05.json -// result_.json -// result.json + const JsonFileName = /^result(_.*)?\.json$/; +const JsonMetaFileName = 'meta.json'; const CsvFileName = /^.*\.csv$/; -async function getDatasetBase(datasetId: string): Promise<{ - datasetType: DatasetType; - basePath: string; - name: string; - jsonFile: string | null; - imageFiles: string[]; - directoryContents: string[]; -}> { - let datasetType: DatasetType = 'image-sequence'; - const exists = fs.existsSync(datasetId); - if (!exists) { - throw new Error(`No dataset exists with path ${datasetId}`); +/** + * getProjectDir returns filepaths to required + * @param settings user settings + * @param datasetId dataset id string + */ +function getProjectDir(settings: Settings, datasetId: string) { + const basePath = npath.join(settings.dataPath, ProjectsFolderName, datasetId); + if (!fs.pathExists(basePath)) { + throw new Error(`missing project directory ${basePath}`); } - const stat = await fs.stat(datasetId); - - if (stat.isDirectory()) { - datasetType = 'image-sequence'; - } else if (stat.isFile()) { - datasetType = 'video'; - } else { - throw new Error('Only regular files and directories are supported'); + const auxPath = npath.join(basePath, AuxFolderName); + if (!fs.pathExists()) { + throw new Error(`missing project aux path ${auxPath}`); } - - let datasetFolderPath = datasetId; - if (datasetType === 'video') { - // get parent folder, since videos reference a file directly - datasetFolderPath = npath.dirname(datasetId); + const metaPath = npath.join(basePath, JsonMetaFileName); + if (!fs.pathExists(metaPath)) { + throw new Error(`missing metadata json file ${metaPath}`); } + let tracksPath = + return { + basePath, + metaPath, + auxPath, + } +} - const contents = await fs.readdir(datasetFolderPath); - const jsonFileCandidates = contents.filter((v) => JsonFileName.test(v)); - let jsonFile = null; +/** + * loadMetadata combines information from JsonFile and directory structure + * to produce a DatasetSchema compliant interface + * @param jsonFile + * @param directoryData + */ +async function _loadMetadata(jsonFile: JsonFileSchema, directoryData: DirectoryData): Promise { + let videoUrl = ''; + let videoPath = ''; + const imageData = [] as FrameImage[]; + const serverInfo = await mediaServerInfo(); - const imageFiles = contents.filter((filename) => { - const abspath = npath.join(datasetFolderPath, filename); + function processFile(abspath: string) { + const basename = npath.basename(abspath); + const abspathuri = `http://localhost:${serverInfo.port}/api/media?path=${abspath}`; const mimetype = mime.lookup(abspath); - if (mimetype && websafeImageTypes.includes(mimetype)) { - return true; + if (mimetype && websafeVideoTypes.includes(mimetype)) { + datasetType = 'video'; + basePath = path.dirname(datasetId); // parent directory of video; + videoPath = abspath; + videoUrl = abspathuri; + } else if (mimetype && websafeImageTypes.includes(mimetype)) { + datasetType = 'image-sequence'; + imageData.push({ + url: abspathuri, + filename: basename, + }); } - return false; - }); + } - if (jsonFileCandidates.length > 1) { - throw new Error('Too many matches for json annotation file!'); - } else if (jsonFileCandidates.length === 1) { - [jsonFile] = jsonFileCandidates; + const info = await fs.stat(datasetId); + + if (info.isDirectory()) { + const contents = await fs.readdir(datasetId); + for (let i = 0; i < contents.length; i += 1) { + processFile(path.join(datasetId, contents[i])); + } + } else { + processFile(datasetId); } - return { - datasetType, - basePath: datasetFolderPath, - jsonFile, - imageFiles, - name: npath.parse(datasetId).name, - directoryContents: contents, - }; + if (datasetType === undefined) { + throw new Error(`Cannot open dataset ${datasetId}: No images or video found`); + } + + return Promise.resolve({ + name: npath.basename(datasetId), + basePath, + videoPath, + meta: { + type: datasetType, + fps: 10, + imageData: datasetType === 'image-sequence' ? imageData : [], + videoUrl: datasetType === 'video' ? videoUrl : undefined, + }, + }); } /** - * Load annotations from JSON + * loadJsonFile processes dataset information from json * @param path a known, existing path */ -async function loadJsonAnnotations(path: string): Promise> { +async function _loadJsonMeta(settings: Settings, datasetId: string): Promise { + const metaFile = npath.join(settings.dataPath, ProjectsFolderName) const rawBuffer = await fs.readFile(path, 'utf-8'); const annotationData = JSON.parse(rawBuffer); - // TODO: validate json schema - return annotationData as Record; + + /** + * Check if this file meets the current schema version + */ + if ('version' in annotationData) { + const { version } = annotationData; + if (version === CurrentSchemaVersion) { + return annotationData as JsonFileSchema; + } + // TODO: schema migration for older schema versions + } + /** + * DEPRECATED schema file with only tracks found, migrate + * to latest schema version. + */ + return { + version: CurrentSchemaVersion, + tracks: annotationData as { [key: string]: TrackData }, + meta: defaultMetadata, + }; } /** * Load detections from disk in priority order - * @param datasetId path + * @param datasetId user data folder name * @param ignoreCSV ignore CSV files if found */ -async function loadDetections(datasetId: string, ignoreCSV = false): - Promise<{ [key: string]: TrackData }> { - const data = {} as { [key: string]: TrackData }; - const base = await getDatasetBase(datasetId); +async function loadDataset(datasetId: string, ignoreCSV = false): Promise { + const meta = /* First, look for a JSON file */ if (base.jsonFile) { - const annotations = loadJsonAnnotations(npath.join(base.basePath, base.jsonFile)); - return annotations; + jsonData = await _loadJsonMeta(npath.join(base.basePath, base.jsonFile), defaultMetadata); } if (ignoreCSV) { @@ -120,8 +168,11 @@ async function loadDetections(datasetId: string, ignoreCSV = false): return data; } - /* return empty by default */ - return Promise.resolve(data); + const ds: DatasetSchema = { + meta: {}, + tracks: {}, + version: CurrentSchemaVersion, + }; } /** @@ -261,10 +312,85 @@ async function postprocess(paths: string[], datasetId: string) { } } -async function loadDataset(): Promise { +async function _initializeAppDataDir(settings: Settings) { + await fs.ensureDir(settings.dataPath); + await fs.ensureDir(npath.join(settings.dataPath, ProjectsFolderName)); + await fs.ensureDir(npath.join(settings.dataPath, TrainedPipelinesFolderName)); +} +async function _initializeProjectDir(settings: Settings, jsonMeta: JsonMeta) { + const projectDir = npath.join(settings.dataPath, ProjectsFolderName, jsonMeta.id); + await _initializeAppDataDir(settings); + await fs.ensureDir(projectDir); } +/** + * importMedia takes in a path and locates as much information as possible + * about the dataset using only the directory structure. + * @param datasetId string path + */ +async function importMedia(settings: Settings, path: string): Promise { + let datasetType: DatasetType = 'image-sequence'; + const exists = fs.existsSync(path); + if (!exists) { + throw new Error(`No dataset exists with path ${path}`); + } + const stat = await fs.stat(path); + + if (stat.isDirectory()) { + datasetType = 'image-sequence'; + } else if (stat.isFile()) { + datasetType = 'video'; + } else { + throw new Error('Only regular files and directories are supported'); + } + + let datasetFolderPath = path; + if (datasetType === 'video') { + // get parent folder, since videos reference a file directly + datasetFolderPath = npath.dirname(path); + } + + const contents = await fs.readdir(datasetFolderPath); + const jsonFileCandidates = contents.filter((v) => JsonFileName.test(v)); + let jsonFile = null; + + const imageFiles = contents.filter((filename) => { + const abspath = npath.join(datasetFolderPath, filename); + const mimetype = mime.lookup(abspath); + if (mimetype && websafeImageTypes.includes(mimetype)) { + return true; + } + return false; + }); + + if (jsonFileCandidates.length > 1) { + throw new Error('Too many matches for json annotation file!'); + } else if (jsonFileCandidates.length === 1) { + [jsonFile] = jsonFileCandidates; + } + + // TODO: parse meta.json if you find it + // TODO: parse FPS from CSV if it exists + + const dsName = npath.parse(path).name; + const dsId = `${cleanString(dsName).substr(0, 20)}_${makeid(10)}`; + const jsonMeta: JsonMeta = { + version: JsonMetaCurrentVersion, + type: datasetType, + id: dsId, + fps: 5, // TODO + originalMediaAbsolutePath: path, + imageFiles, + name: dsName, + }; + + await _initializeProjectDir(settings, jsonMeta); + + postprocess() +} + + async function openLink(url: string) { shell.openExternal(url); } @@ -273,7 +399,6 @@ export default { openLink, getAuxFolder, createKwiverRunWorkingDir, - getDatasetBase, getPipelineList, loadDetections, saveDetections, diff --git a/client/platform/desktop/backend/platforms/linux.ts b/client/platform/desktop/backend/platforms/linux.ts index b10875883..1edbc3541 100644 --- a/client/platform/desktop/backend/platforms/linux.ts +++ b/client/platform/desktop/backend/platforms/linux.ts @@ -53,10 +53,11 @@ async function validateViamePath(settings: Settings): Promise { * @param settings global settings */ async function runPipeline( + settings: Settings, runPipelineArgs: RunPipeline, updater: (msg: DesktopJobUpdate) => void, ): Promise { - const { settings, datasetId, pipelineName } = runPipelineArgs; + const { datasetId, pipelineName } = runPipelineArgs; const isValid = await validateViamePath(settings); if (isValid !== true) { throw new Error(isValid); diff --git a/client/platform/desktop/backend/platforms/utils.ts b/client/platform/desktop/backend/platforms/utils.ts new file mode 100644 index 000000000..22de84347 --- /dev/null +++ b/client/platform/desktop/backend/platforms/utils.ts @@ -0,0 +1,24 @@ + +/** + * Get a nice safe string + */ +function cleanString(dirty: string) { + return dirty.replace(/[^a-z0-9]/gi, '_').toLowerCase(); +} + +// https://stackoverflow.com/questions/1349404/generate-random-string-characters-in-javascript +function makeid(length: number): string { + let result = ''; + const characters = 'abcdefghijklmnopqrstuvwxyz0123456789'; + const charactersLength = characters.length; + for (let i = 0; i < length; i += 1) { + result += characters.charAt(Math.floor(Math.random() * charactersLength)); + } + return result; +} + +// eslint-disable-next-line import/prefer-default-export +export { + cleanString, + makeid, +}; diff --git a/client/platform/desktop/backend/platforms/windows.ts b/client/platform/desktop/backend/platforms/windows.ts index 9979a65cc..f15d696cd 100644 --- a/client/platform/desktop/backend/platforms/windows.ts +++ b/client/platform/desktop/backend/platforms/windows.ts @@ -68,10 +68,11 @@ async function validateViamePath(settings: Settings): Promise { * @param settings global settings */ async function runPipeline( + settings: Settings, runPipelineArgs: RunPipeline, updater: (msg: DesktopJobUpdate) => void, ): Promise { - const { settings, datasetId, pipelineName } = runPipelineArgs; + const { datasetId, pipelineName } = runPipelineArgs; const isValid = await validateViamePath(settings); if (isValid !== true) { throw new Error(isValid); diff --git a/client/platform/desktop/backend/server.ts b/client/platform/desktop/backend/server.ts index f3cb622ef..aef1b1fc7 100644 --- a/client/platform/desktop/backend/server.ts +++ b/client/platform/desktop/backend/server.ts @@ -47,6 +47,21 @@ router.register('/api', withErrorHandler((req, res) => { res.end(); })); +router.register('/api/dataset', withErrorHandler((req, res) => { + const { url } = req; + if (!url) { + throw new Error('Impossible scenario, req.url was empty'); + } + const parsedurl = parser.parse(url, true); + let datasetId = parsedurl.query ? parsedurl.query.datasetId : undefined; + + if (datasetId === undefined || Array.isArray(datasetId)) { + return fail(res, 404, `Invalid dataset ID: ${datasetId}`); + } + + datasetId = decodeURI(datasetId); +})); + router.register('/api/media', withErrorHandler((req, res) => { const { url } = req; if (!url) { diff --git a/client/platform/desktop/background.ts b/client/platform/desktop/background.ts index c70995428..2c38a1939 100644 --- a/client/platform/desktop/background.ts +++ b/client/platform/desktop/background.ts @@ -38,8 +38,7 @@ function createWindow() { // Use pluginOptions.nodeIntegration, leave this alone // See nklayman.github.io/vue-cli-plugin-electron-builder/guide/security.html // #node-integration for more info - nodeIntegration: (process.env - .ELECTRON_NODE_INTEGRATION as unknown) as boolean, + nodeIntegration: (!!process.env.ELECTRON_NODE_INTEGRATION), plugins: true, enableRemoteModule: true, }, diff --git a/client/platform/desktop/constants.ts b/client/platform/desktop/constants.ts index 54ab0a034..c30a5cb6b 100644 --- a/client/platform/desktop/constants.ts +++ b/client/platform/desktop/constants.ts @@ -1,4 +1,6 @@ -import { DatasetMeta } from 'viame-web-common/apispec'; +import type { + DatasetMeta, DatasetMetaMutable, DatasetSchema, DatasetType, +} from 'viame-web-common/apispec'; export const websafeVideoTypes = [ 'video/mp4', @@ -15,7 +17,9 @@ export const websafeImageTypes = [ // 'image/webp', ]; +export const JsonMetaCurrentVersion = 1; export const SettingsCurrentVersion = 1; + export interface Settings { // version a schema version version: number; @@ -25,15 +29,32 @@ export interface Settings { dataPath: string; } -export interface DesktopDataset { - // name filename (for video) or folder name (for images) +/** + * JsonMetadata is a SUBSET of DatasetMeta contained within + * the JsonFileSchema. The remaining parts of DatasetMeta must + * be calculated at load time. + */ +export interface JsonMeta extends DatasetMetaMutable { + // version used to manage schema migrations + version: number; + // immutable dataset type + type: DatasetType; + // immutable datset identifier + id: string; + // this will become mutable in the future. + fps: number; + // the original name derived from media path name: string; - // basePath path of dataset working directory - basePath: string; - // vidoPath path of single video file - videoPath?: string; - // meta DatasetMeta - meta: DatasetMeta; + // may point to media outside the project folder + originalMediaAbsolutePath: string; + // points to media inside the project folder + transcodedMediaRelativePath?: string; + // ordered image filenames IF this is an image dataset + imageFiles: string[]; +} + +export interface DesktopDataset extends DatasetSchema { + meta: JsonMeta & DatasetMeta; // JsonMeta satisfies DatasetMeta } interface NvidiaSmiTextRecord { @@ -84,5 +105,4 @@ export interface DesktopJobUpdate extends DesktopJob { export interface RunPipeline { datasetId: string; pipelineName: string; - settings: Settings; } diff --git a/client/platform/desktop/store/index.ts b/client/platform/desktop/store/index.ts index 0c0ac41e3..1fd1313f9 100644 --- a/client/platform/desktop/store/index.ts +++ b/client/platform/desktop/store/index.ts @@ -1,4 +1,4 @@ -import { Api, Pipe } from 'viame-web-common/apispec'; +import { Api } from 'viame-web-common/apispec'; import * as api from '../api/main'; import { settings } from './settings'; @@ -19,7 +19,7 @@ export default function wrap(): Api { return api.getPipelineList(settings.value); } - async function runPipeline(itemId: string, pipeline: Pipe) { + async function runPipeline(itemId: string, pipeline: string) { const job = await api.runPipeline(itemId, pipeline, settings.value); const datasets = job.datasetIds.map(((id) => getDataset(id).value)); getOrCreateHistory(job, datasets); @@ -27,7 +27,6 @@ export default function wrap(): Api { return { ...api, - loadMetadata, getPipelineList, runPipeline, }; diff --git a/client/platform/desktop/store/settings.ts b/client/platform/desktop/store/settings.ts index 06fc7c7df..b5c3ddd67 100644 --- a/client/platform/desktop/store/settings.ts +++ b/client/platform/desktop/store/settings.ts @@ -45,10 +45,13 @@ async function init() { // pass } settings.value = settingsvalue; + + ipcRenderer.send('update-settings', settings.value); } async function setSettings(s: Settings) { window.localStorage.setItem(SettingsKey, JSON.stringify(s)); + ipcRenderer.send('update-settings', settings.value); } // Will be initialized on first import diff --git a/client/viame-web-common/apispec.ts b/client/viame-web-common/apispec.ts index 20c8d97ef..0486cf870 100644 --- a/client/viame-web-common/apispec.ts +++ b/client/viame-web-common/apispec.ts @@ -4,8 +4,6 @@ import { use } from 'vue-media-annotator/provides'; import Track, { TrackData, TrackId } from 'vue-media-annotator/track'; import { CustomStyle } from 'vue-media-annotator/use/useStyling'; -const ApiSymbol = Symbol('api'); - type DatasetType = 'image-sequence' | 'video'; interface Attribute { @@ -27,7 +25,7 @@ interface Category { pipes: Pipe[]; } -export interface TrainingConfigs { +interface TrainingConfigs { configs: string[]; default: string; } @@ -44,26 +42,27 @@ interface FrameImage { filename: string; } +/** + * The parts of metadata a user should be able to modify. + */ interface DatasetMetaMutable { customTypeStyling?: Record; confidenceFilters?: Record; } -export interface DatasetMeta extends DatasetMetaMutable { - type: Readonly; - fps: Readonly; +interface DatasetMeta extends DatasetMetaMutable { imageData: FrameImage[]; videoUrl: string | undefined; + type: Readonly; + fps: Readonly; // this will become mutable in the future. } - /** * DatasetSchema is a structure that describes everything about * media that could be opened in DIVE. This schema is JSON * serializable (no maps, sets, or classes in the tree) */ -export interface DatasetSchema { - version: number; +interface DatasetSchema { // TODO: in a future version attributes will be part of the dataset schema // attributes: Attribute[]; meta: DatasetMeta; @@ -89,6 +88,8 @@ interface Api { saveMetadata(datasetId: string, metadata: DatasetMetaMutable): Promise; } +const ApiSymbol = Symbol('api'); + /** * provideApi specifies an implementation of the data persistence interface * for use in vue-web-common @@ -103,15 +104,20 @@ function useApi() { } export { + provideApi, + useApi, +}; + +export type { Api, Attribute, DatasetMeta, DatasetMetaMutable, + DatasetSchema, DatasetType, FrameImage, Pipe, Pipelines, SaveDetectionsArgs, - provideApi, - useApi, + TrainingConfigs, }; From 0d3b13ec98af4dd045cc57e01f9c3308aa7dbbfc Mon Sep 17 00:00:00 2001 From: Brandon Davis Date: Wed, 23 Dec 2020 14:30:34 -0500 Subject: [PATCH 3/8] wip --- .../desktop/backend/platforms/common.ts | 368 +++++++++--------- client/platform/desktop/backend/server.ts | 12 + client/platform/desktop/constants.ts | 32 +- client/viame-web-common/apispec.ts | 4 +- 4 files changed, 233 insertions(+), 183 deletions(-) diff --git a/client/platform/desktop/backend/platforms/common.ts b/client/platform/desktop/backend/platforms/common.ts index 1abd0adcb..f58a3131b 100644 --- a/client/platform/desktop/backend/platforms/common.ts +++ b/client/platform/desktop/backend/platforms/common.ts @@ -6,17 +6,19 @@ import fs from 'fs-extra'; import { shell } from 'electron'; import mime from 'mime-types'; import moment from 'moment'; -import { TrackData } from 'vue-media-annotator/track'; +import type { TrackData } from 'vue-media-annotator/track'; import { - DatasetType, Pipelines, DatasetSchema, SaveDetectionsArgs, FrameImage, + DatasetType, MultiTrackRecord, Pipelines, SaveDetectionsArgs, FrameImage, } from 'viame-web-common/apispec'; import { - JsonMeta, Settings, websafeImageTypes, websafeVideoTypes, JsonMetaCurrentVersion, DesktopDataset, + JsonMeta, Settings, websafeImageTypes, websafeVideoTypes, JsonMetaCurrentVersion, DesktopDataset, otherImageTypes, } from 'platform/desktop/constants'; import * as viameSerializers from 'platform/desktop/backend/serializers/viame'; +import { makeMediaUrl } from '../server'; import { cleanString, makeid } from './utils'; +import { number } from 'yargs'; const ProjectsFolderName = 'DIVE_Projects'; const TrainedPipelinesFolderName = 'DIVE_Trained_Pipelines'; @@ -28,169 +30,140 @@ const JsonMetaFileName = 'meta.json'; const CsvFileName = /^.*\.csv$/; /** - * getProjectDir returns filepaths to required - * @param settings user settings - * @param datasetId dataset id string - */ -function getProjectDir(settings: Settings, datasetId: string) { - const basePath = npath.join(settings.dataPath, ProjectsFolderName, datasetId); - if (!fs.pathExists(basePath)) { - throw new Error(`missing project directory ${basePath}`); - } - const auxPath = npath.join(basePath, AuxFolderName); - if (!fs.pathExists()) { - throw new Error(`missing project aux path ${auxPath}`); - } - const metaPath = npath.join(basePath, JsonMetaFileName); - if (!fs.pathExists(metaPath)) { - throw new Error(`missing metadata json file ${metaPath}`); - } - let tracksPath = - return { - basePath, - metaPath, - auxPath, - } -} - -/** - * loadMetadata combines information from JsonFile and directory structure - * to produce a DatasetSchema compliant interface - * @param jsonFile - * @param directoryData + * locate json track file in a directory + * @param path path to a directory + * @returns absolute path to json file or null */ -async function _loadMetadata(jsonFile: JsonFileSchema, directoryData: DirectoryData): Promise { - let videoUrl = ''; - let videoPath = ''; - const imageData = [] as FrameImage[]; - const serverInfo = await mediaServerInfo(); - const contents = await fs.readdir(datasetFolderPath); - - function processFile(abspath: string) { - const basename = npath.basename(abspath); - const abspathuri = `http://localhost:${serverInfo.port}/api/media?path=${abspath}`; - const mimetype = mime.lookup(abspath); - if (mimetype && websafeVideoTypes.includes(mimetype)) { - datasetType = 'video'; - basePath = path.dirname(datasetId); // parent directory of video; - videoPath = abspath; - videoUrl = abspathuri; - } else if (mimetype && websafeImageTypes.includes(mimetype)) { - datasetType = 'image-sequence'; - imageData.push({ - url: abspathuri, - filename: basename, - }); - } - } - - const info = await fs.stat(datasetId); - - if (info.isDirectory()) { - const contents = await fs.readdir(datasetId); - for (let i = 0; i < contents.length; i += 1) { - processFile(path.join(datasetId, contents[i])); - } - } else { - processFile(datasetId); - } - +async function _findJsonTrackFile(basePath: string): Promise { + const contents = await fs.readdir(basePath); const jsonFileCandidates: string[] = []; await Promise.all(contents.map(async (name) => { if (JsonFileName.test(name)) { - const fullPath = npath.join(datasetFolderPath, name); + const fullPath = npath.join(basePath, name); const statResult = await fs.stat(fullPath); if (statResult.isFile()) { - jsonFileCandidates.push(name); + jsonFileCandidates.push(fullPath); } } })); - - let jsonFile = null; if (jsonFileCandidates.length > 1) { - throw new Error('Too many matches for json annotation file!'); + throw new Error(`too many matches for json annotation file in ${basePath}`); } else if (jsonFileCandidates.length === 1) { - [jsonFile] = jsonFileCandidates; + return jsonFileCandidates[0]; + } + return null; +} + +/** + * _getProjectDir returns filepaths to required members of a dataset project directory. + * + * REQUIRED members: meta.json, results*.json + * + * OPTIONAL members: aux/ will be created if none exists + * + * @param settings user settings + * @param datasetId dataset id string + */ +async function _getProjectDir(settings: Settings, datasetId: string) { + const basePath = npath.join(settings.dataPath, ProjectsFolderName, datasetId); + if (!fs.pathExistsSync(basePath)) { + throw new Error(`missing project directory ${basePath}`); + } + + const auxDirAbsPath = npath.join(basePath, AuxFolderName); + fs.ensureDirSync(auxDirAbsPath); + + const metaFileAbsPath = npath.join(basePath, JsonMetaFileName); + if (!fs.pathExists(metaFileAbsPath)) { + throw new Error(`missing metadata json file ${metaFileAbsPath}`); } - if (datasetType === undefined) { - throw new Error(`Cannot open dataset ${datasetId}: No images or video found`); + const trackFileAbsPath = await _findJsonTrackFile(basePath); + if (trackFileAbsPath === null) { + throw new Error(`missing track json file in ${basePath}`); } - return Promise.resolve({ - name: npath.basename(datasetId), + return { + auxDirAbsPath, basePath, - videoPath, - meta: { - type: datasetType, - fps: 10, - imageData: datasetType === 'image-sequence' ? imageData : [], - videoUrl: datasetType === 'video' ? videoUrl : undefined, - }, - }); + metaFileAbsPath, + trackFileAbsPath, + }; } /** - * loadJsonFile processes dataset information from json - * @param path a known, existing path + * _loadJsonMeta processes dataset information from json + * @param metaPath a known, existing path */ -async function _loadJsonMeta(settings: Settings, datasetId: string): Promise { - const metaFile = npath.join(settings.dataPath, ProjectsFolderName) - const rawBuffer = await fs.readFile(path, 'utf-8'); - const annotationData = JSON.parse(rawBuffer); - - /** - * Check if this file meets the current schema version - */ - if ('version' in annotationData) { - const { version } = annotationData; - if (version === CurrentSchemaVersion) { - return annotationData as JsonFileSchema; +async function _loadJsonMeta(metaAbsPath: string): Promise { + const rawBuffer = await fs.readFile(metaAbsPath, 'utf-8'); + const metaJson = JSON.parse(rawBuffer); + /* check if this file meets the current schema version */ + if ('version' in metaJson) { + const { version } = metaJson; + if (version !== JsonMetaCurrentVersion) { + // TODO: schema migration for older schema versions + throw new Error('outdated meta schema version found, migration not implemented'); } - // TODO: schema migration for older schema versions } - /** - * DEPRECATED schema file with only tracks found, migrate - * to latest schema version. - */ - return { - version: CurrentSchemaVersion, - tracks: annotationData as { [key: string]: TrackData }, - meta: defaultMetadata, - }; + return metaJson as JsonMeta; } /** - * Load detections from disk in priority order - * @param datasetId user data folder name - * @param ignoreCSV ignore CSV files if found + * _loadJsonTracks load from file + * @param tracksPath a known, existing path */ -async function loadDataset(datasetId: string, ignoreCSV = false): Promise { - const meta = - - /* First, look for a JSON file */ - if (base.jsonFile) { - jsonData = await _loadJsonMeta(npath.join(base.basePath, base.jsonFile), defaultMetadata); +async function _loadJsonTracks(tracksAbsPath: string): Promise { + const rawBuffer = await fs.readFile(tracksAbsPath, 'utf-8'); + const annotationData = JSON.parse(rawBuffer) as MultiTrackRecord; + // TODO: somehow verify the schema of this file + if (Array.isArray(annotationData)) { + throw new Error('object expected in track json'); } + return annotationData; +} - if (ignoreCSV) { - return Promise.resolve(data); - } +/** + * loadDataset load detections and meta from disk + * @param settings user settings + * @param datasetId user data folder name + */ +async function loadDataset(settings: Settings, datasetId: string): Promise { + const projectDirData = await _getProjectDir(settings, datasetId); + const projectMetaData = await _loadJsonMeta(projectDirData.metaFileAbsPath); - /* Then, look for a CSV */ - const csvFileCandidates = base.directoryContents.filter((v) => CsvFileName.test(v)); - if (csvFileCandidates.length === 1) { - const tracks = await viameSerializers.parseFile( - npath.join(base.basePath, csvFileCandidates[0]), - ); - tracks.forEach((t) => { data[t.trackId.toString()] = t; }); - return data; + let videoUrl = ''; + let imageData = [] as FrameImage[]; + + /* Generate URLs against embedded media server from known file paths on disk */ + if (projectMetaData.type === 'video') { + /* If the video has been transcoded, use that video */ + if (projectMetaData.transcodedVideoFile) { + videoUrl = makeMediaUrl( + npath.join(projectDirData.basePath, projectMetaData.transcodedVideoFile), + ); + } else { + videoUrl = makeMediaUrl( + npath.join(projectMetaData.originalBasePath, projectMetaData.originalVideoFile), + ); + } + } else if (projectMetaData.type === 'image-sequence') { + /* TODO: if images were transcoded, use them */ + imageData = projectMetaData.originalImageFiles.map((filename: string) => ({ + url: makeMediaUrl(npath.join(projectMetaData.originalBasePath, filename)), + filename, + })); + } else { + throw new Error(`unexpected project type for id="${datasetId}" type="${projectMetaData.type}"`); } - const ds: DatasetSchema = { - meta: {}, - tracks: {}, - version: CurrentSchemaVersion, + return { + meta: { + ...projectMetaData, + videoUrl, + imageData, + }, + tracks: await _loadJsonTracks(projectDirData.trackFileAbsPath), }; } @@ -262,13 +235,12 @@ async function createKwiverRunWorkingDir(datasetName: string, baseDir: string, p } /** - * Save pre-serialized tracks to disk - * @param datasetId path + * _saveSerialized save pre-serialized tracks to disk * @param trackData json serialized track object */ -async function saveSerialized( +async function _saveSerialized( datasetId: string, - trackData: Record, + trackData: MultiTrackRecord, ) { const time = moment().format('MM-DD-YYYY_HH-MM-SS'); const newFileName = `result_${time}.json`; @@ -295,25 +267,31 @@ async function saveSerialized( * @param datasetId path * @param args save args */ -async function saveDetections(datasetId: string, args: SaveDetectionsArgs) { +async function saveDetections(settings: Settings, datasetId: string, args: SaveDetectionsArgs) { /* Update existing track file */ - const existing = await loadDetections(datasetId, true); + const projectDirInfo = await _getProjectDir(settings, datasetId); + const existing = await _loadJsonTracks(projectDirInfo.trackFileAbsPath); args.delete.forEach((trackId) => delete existing[trackId.toString()]); args.upsert.forEach((track, trackId) => { existing[trackId.toString()] = track.serialize(); }); - return saveSerialized(datasetId, existing); + return _saveSerialized(datasetId, existing); } /** - * Postprocess possible annotation files - * @param paths paths to input annotation files in descending priority order. - * Only the first successful input will be loaded. + * processOtherAnnotationFiles imports data from external annotation formats + * + * Only VIAME CSV is currently supported. + * + * @param paths paths to possible input annotation files * @param datasetId dataset id path */ -async function postprocess(paths: string[], datasetId: string) { - for (let i = 0; i < paths.length; i += 1) { - const path = paths[i]; +async function processOtherAnnotationFiles( + absPaths: string[], datasetId: string, +): Promise<{ fps?: number }> { + const fps = undefined; + for (let i = 0; i < absPaths.length; i += 1) { + const path = absPaths[i]; if (!fs.existsSync(path)) { // eslint-disable-next-line no-continue continue; @@ -325,10 +303,11 @@ async function postprocess(paths: string[], datasetId: string) { const data = {} as Record; tracks.forEach((t) => { data[t.trackId.toString()] = t; }); // eslint-disable-next-line no-await-in-loop - await saveSerialized(datasetId, data); + await _saveSerialized(datasetId, data); break; // Exit on first successful detection load } } + return { fps }; } async function _initializeAppDataDir(settings: Settings) { @@ -337,25 +316,33 @@ async function _initializeAppDataDir(settings: Settings) { await fs.ensureDir(npath.join(settings.dataPath, TrainedPipelinesFolderName)); } -async function _initializeProjectDir(settings: Settings, jsonMeta: JsonMeta) { +/** + * Intialize a new project directory + * @returns absolute path to new project dcirectory + */ +async function _initializeProjectDir(settings: Settings, jsonMeta: JsonMeta): string { const projectDir = npath.join(settings.dataPath, ProjectsFolderName, jsonMeta.id); await _initializeAppDataDir(settings); await fs.ensureDir(projectDir); + return projectDir; } /** - * importMedia takes in a path and locates as much information as possible - * about the dataset using only the directory structure. - * @param datasetId string path + * importMedia locates as much information as possible + * about a dataset using only the directory structure. + * @param settings user settings + * @param path path to import dir/file + * @returns datasetId */ -async function importMedia(settings: Settings, path: string): Promise { - let datasetType: DatasetType = 'image-sequence'; +async function importMedia(settings: Settings, path: string): Promise { + let datasetType: DatasetType; + const exists = fs.existsSync(path); if (!exists) { - throw new Error(`No dataset exists with path ${path}`); + throw new Error(`file or directory not found: ${path}`); } - const stat = await fs.stat(path); + const stat = await fs.stat(path); if (stat.isDirectory()) { datasetType = 'image-sequence'; } else if (stat.isFile()) { @@ -364,36 +351,47 @@ async function importMedia(settings: Settings, path: string): Promise throw new Error('Only regular files and directories are supported'); } + const dsName = npath.parse(path).name; + const dsId = `${cleanString(dsName).substr(0, 20)}_${makeid(10)}`; let datasetFolderPath = path; if (datasetType === 'video') { // get parent folder, since videos reference a file directly datasetFolderPath = npath.dirname(path); } + let imageFiles: string[] = []; + const transcodedImageFiles: string[] = []; // TODO: unused + let videoFile = ''; + const transcodedVideoFile = ''; // TODO: unused const contents = await fs.readdir(datasetFolderPath); - const jsonFileCandidates = contents.filter((v) => JsonFileName.test(v)); - let jsonFile = null; - - const imageFiles = contents.filter((filename) => { - const abspath = npath.join(datasetFolderPath, filename); - const mimetype = mime.lookup(abspath); - if (mimetype && websafeImageTypes.includes(mimetype)) { - return true; - } - return false; - }); - if (jsonFileCandidates.length > 1) { - throw new Error('Too many matches for json annotation file!'); - } else if (jsonFileCandidates.length === 1) { - [jsonFile] = jsonFileCandidates; + /* Extract and validate media from import path */ + if (datasetType === 'video') { + videoFile = npath.basename(path); + const mimetype = mime.lookup(path); + if (mimetype) { + if (websafeImageTypes.includes(mimetype) || otherImageTypes.includes(mimetype)) { + throw new Error('User chose image file for video import option'); + } else if (websafeVideoTypes.includes(mimetype)) { + /* TODO: Kick off video inspection and maybe transcode */ + } + } else { + throw new Error(`Could not determine video MIME type for ${path}`); + } + } else if (datasetType === 'image-sequence') { + imageFiles = contents.filter((filename) => { + const abspath = npath.join(datasetFolderPath, filename); + const mimetype = mime.lookup(abspath); + /* TODO: support transcoding of non-web-safe image types */ + return !!(mimetype && websafeImageTypes.includes(mimetype)); + }); + } else { + throw new Error('Only video and image-sequence types are supported'); } - // TODO: parse meta.json if you find it + // TODO: parse FPS from CSV if it exists - const dsName = npath.parse(path).name; - const dsId = `${cleanString(dsName).substr(0, 20)}_${makeid(10)}`; const jsonMeta: JsonMeta = { version: JsonMetaCurrentVersion, type: datasetType, @@ -404,9 +402,25 @@ async function importMedia(settings: Settings, path: string): Promise name: dsName, }; - await _initializeProjectDir(settings, jsonMeta); + const projectDirAbsPath = await _initializeProjectDir(settings, jsonMeta); + + /* Look for JSON track file */ + const trackFileAbsPath = await _findJsonTrackFile(datasetFolderPath); + if (trackFileAbsPath !== null) { + /* Move the track file into the new project directory */ + await fs.move( + trackFileAbsPath, + npath.join(projectDirAbsPath, npath.basename(trackFileAbsPath)), + ); + /* Look for other supported annotation types */ + } else { + const csvFileCandidates = contents + .filter((v) => CsvFileName.test(v)) + .map((filename) => npath.join(datasetFolderPath, filename)); + const { fps } = await processOtherAnnotationFiles(csvFileCandidates, dsId); + + } - postprocess() } diff --git a/client/platform/desktop/backend/server.ts b/client/platform/desktop/backend/server.ts index aef1b1fc7..93f47d3f2 100644 --- a/client/platform/desktop/backend/server.ts +++ b/client/platform/desktop/backend/server.ts @@ -1,4 +1,5 @@ import mime from 'mime-types'; +import { AddressInfo } from 'net'; import pump from 'pump'; import rangeParser from 'range-parser'; import http from 'http'; @@ -129,4 +130,15 @@ const server = http.createServer((req, res) => { handler.process(req, res); }); +/** + * makeMediaUrl gets a URL for the given file path + */ +export function makeMediaUrl(filepath: string) { + const addr = server.address() as AddressInfo | null; + if (!addr) { + throw new Error('server has not initialized yet'); + } + return `http://localhost:${addr.port}/api/media?path=${filepath}`; +} + export default server; diff --git a/client/platform/desktop/constants.ts b/client/platform/desktop/constants.ts index c30a5cb6b..c80e69fc9 100644 --- a/client/platform/desktop/constants.ts +++ b/client/platform/desktop/constants.ts @@ -17,6 +17,11 @@ export const websafeImageTypes = [ // 'image/webp', ]; +export const otherSupportedImageTypes = [ + 'image/avif', + 'image/tiff', +]; + export const JsonMetaCurrentVersion = 1; export const SettingsCurrentVersion = 1; @@ -37,20 +42,37 @@ export interface Settings { export interface JsonMeta extends DatasetMetaMutable { // version used to manage schema migrations version: number; + // immutable dataset type type: DatasetType; + // immutable datset identifier id: string; + // this will become mutable in the future. fps: number; + // the original name derived from media path name: string; - // may point to media outside the project folder - originalMediaAbsolutePath: string; - // points to media inside the project folder - transcodedMediaRelativePath?: string; + + // absolute base path on disk where dataset was imported from + originalBasePath: string; + + // video file path + // relateive to originalBasePath + originalVideoFile: string; + + // output of web safe transcoding + // relative to project path + transcodedVideoFile?: string; + // ordered image filenames IF this is an image dataset - imageFiles: string[]; + // relative to originalBasePath + originalImageFiles: string[]; + + // ordered image filenames of transcoded images + // relative to project path + transcodedImageFiles: string[]; } export interface DesktopDataset extends DatasetSchema { diff --git a/client/viame-web-common/apispec.ts b/client/viame-web-common/apispec.ts index 0486cf870..780013ea1 100644 --- a/client/viame-web-common/apispec.ts +++ b/client/viame-web-common/apispec.ts @@ -5,6 +5,7 @@ import Track, { TrackData, TrackId } from 'vue-media-annotator/track'; import { CustomStyle } from 'vue-media-annotator/use/useStyling'; type DatasetType = 'image-sequence' | 'video'; +type MultiTrackRecord = Record; interface Attribute { belongs: 'track' | 'detection'; @@ -66,7 +67,7 @@ interface DatasetSchema { // TODO: in a future version attributes will be part of the dataset schema // attributes: Attribute[]; meta: DatasetMeta; - tracks: { [key: string]: TrackData }; + tracks: MultiTrackRecord; } interface Api { @@ -116,6 +117,7 @@ export type { DatasetSchema, DatasetType, FrameImage, + MultiTrackRecord, Pipe, Pipelines, SaveDetectionsArgs, From 9e87a2c18fc35fc8427407c9f08c31f2ffa147c4 Mon Sep 17 00:00:00 2001 From: Brandon Davis Date: Tue, 29 Dec 2020 10:27:08 -0500 Subject: [PATCH 4/8] Refactor desktop project data tracking to custom directory structure --- client/package.json | 3 +- client/platform/desktop/api/main.ts | 38 +-- client/platform/desktop/backend/ipcService.ts | 41 ++- .../desktop/backend/platforms/common.ts | 280 ++++++++++-------- .../desktop/backend/platforms/linux.ts | 31 +- .../desktop/backend/platforms/windows.ts | 31 +- client/platform/desktop/backend/server.ts | 28 +- .../desktop/backend/state/settings.ts | 19 ++ client/platform/desktop/components/Jobs.vue | 6 +- client/platform/desktop/components/Recent.vue | 22 +- .../platform/desktop/components/Settings.vue | 36 ++- .../desktop/components/ViewerLoader.vue | 10 +- client/platform/desktop/constants.ts | 22 +- client/platform/desktop/main.ts | 2 + client/platform/desktop/router.ts | 2 +- client/platform/desktop/store/dataset.ts | 25 +- client/platform/desktop/store/index.ts | 56 +++- client/platform/web-girder/App.vue | 17 +- client/platform/web-girder/store/Dataset.ts | 15 +- client/viame-web-common/apispec.ts | 2 +- client/viame-web-common/components/Viewer.vue | 54 ++-- client/vue.config.js | 1 + 22 files changed, 417 insertions(+), 324 deletions(-) create mode 100644 client/platform/desktop/backend/state/settings.ts diff --git a/client/package.json b/client/package.json index 3fbac0f4f..e05c94a9a 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "vue-media-annotator", - "version": "1.3.1", + "version": "1.3.2", "author": { "name": "Kitware, Inc.", "email": "viame-web@kitware.com" @@ -88,6 +88,7 @@ "eslint-import-resolver-typescript": "^2.2.0", "eslint-plugin-import": "^2.20.2", "eslint-plugin-vue": "^6.2.2", + "express": "^4.17.1", "fs-extra": "^9.0.1", "git-describe": "^4.0.4", "jest": "^26.0.1", diff --git a/client/platform/desktop/api/main.ts b/client/platform/desktop/api/main.ts index dfa827b45..1ce69b96f 100644 --- a/client/platform/desktop/api/main.ts +++ b/client/platform/desktop/api/main.ts @@ -1,23 +1,19 @@ +import { AddressInfo } from 'net'; import type { FileFilter } from 'electron'; import { ipcRenderer, remote } from 'electron'; -import { - Attribute, - DatasetMetaMutable, - DatasetType, Pipelines, TrainingConfigs, +import type { + Attribute, DatasetType, Pipe, Pipelines, TrainingConfigs, } from 'viame-web-common/apispec'; -import common from '../backend/platforms/common'; import { - DesktopJob, NvidiaSmiReply, RunPipeline, - websafeVideoTypes, Settings, -} from '../constants'; - - -const { loadDetections, saveDetections } = common; - + DesktopJob, NvidiaSmiReply, RunPipeline, websafeVideoTypes, +} from 'platform/desktop/constants'; +function mediaServerInfo(): Promise { + return ipcRenderer.invoke('server-info'); +} function nvidiaSmi(): Promise { return ipcRenderer.invoke('nvidia-smi'); @@ -55,8 +51,8 @@ async function deleteAttribute(data: Attribute) { return Promise.resolve([] as Attribute[]); } -async function getPipelineList(settings: Settings): Promise { - return ipcRenderer.invoke('get-pipeline-list', settings); +async function getPipelineList(): Promise { + return ipcRenderer.invoke('get-pipeline-list'); } // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -71,16 +67,10 @@ async function runTraining( return Promise.resolve(); } -// eslint-disable-next-line -async function saveMetadata(datasetId: string, metadata: DatasetMetaMutable) { - return Promise.resolve(); -} - -async function runPipeline(itemId: string, pipeline: string, settings: Settings) { +async function runPipeline(itemId: string, pipeline: Pipe) { const args: RunPipeline = { - pipelineName: pipeline, + pipeline, datasetId: itemId, - settings, }; const job: DesktopJob = await ipcRenderer.invoke('run-pipeline', args); return job; @@ -95,11 +85,9 @@ export { runPipeline, getTrainingConfigurations, runTraining, - loadDetections, - saveDetections, - saveMetadata, /* Nonstandard APIs */ openFromDisk, openLink, nvidiaSmi, + mediaServerInfo, }; diff --git a/client/platform/desktop/backend/ipcService.ts b/client/platform/desktop/backend/ipcService.ts index d2d6f90b1..0e1f2da8b 100644 --- a/client/platform/desktop/backend/ipcService.ts +++ b/client/platform/desktop/backend/ipcService.ts @@ -1,47 +1,45 @@ import OS from 'os'; import { ipcMain } from 'electron'; -import { DesktopJobUpdate, RunPipeline, Settings } from '../constants'; +import { DesktopJobUpdate, RunPipeline, Settings } from 'platform/desktop/constants'; + +import server from './server'; import linux from './platforms/linux'; import win32 from './platforms/windows'; import common from './platforms/common'; +import settings from './state/settings'; -let settings: Settings; - -function getSetting() { - if (settings === undefined) { - throw new Error('Settings has not been initialized!'); - } - return settings; +// defaults to linux if win32 doesn't exist +const currentPlatform = OS.platform() === 'win32' ? win32 : linux; +if (OS.platform() === 'win32') { + win32.initialize(); } export default function register() { /** * Platform-agnostic methods */ - + ipcMain.handle('server-info', async () => server.address()); ipcMain.handle('get-pipeline-list', async () => { - const ret = await common.getPipelineList(getSetting()); + const ret = await common.getPipelineList(settings.get()); return ret; }); ipcMain.handle('open-link-in-browser', (_, url: string) => { common.openLink(url); }); ipcMain.on('update-settings', async (_, s: Settings) => { - settings = s; + settings.set(s); + }); + ipcMain.handle('import-media', async (_, path: string) => { + const ret = await common.importMedia(settings.get(), path); + return ret; }); /** * Platform-dependent methods */ - // defaults to linux if win32 doesn't exist - const currentPlatform = OS.platform() === 'win32' ? win32 : linux; - - if (OS.platform() === 'win32') { - win32.initialize(); - } ipcMain.handle('nvidia-smi', async () => { const ret = await currentPlatform.nvidiaSmi(); return ret; @@ -50,17 +48,14 @@ export default function register() { const defaults = currentPlatform.DefaultSettings; return defaults; }); - ipcMain.handle('validate-settings', async () => { - const ret = await currentPlatform.validateViamePath(getSetting()); + ipcMain.handle('validate-settings', async (_, s: Settings) => { + const ret = await currentPlatform.validateViamePath(s); return ret; - }); - ipcMain.handle('import-media', async () => { - }); ipcMain.handle('run-pipeline', async (event, args: RunPipeline) => { const updater = (update: DesktopJobUpdate) => { event.sender.send('job-update', update); }; - return currentPlatform.runPipeline(getSetting(), args, updater); + return currentPlatform.runPipeline(settings.get(), args, updater); }); } diff --git a/client/platform/desktop/backend/platforms/common.ts b/client/platform/desktop/backend/platforms/common.ts index f58a3131b..0d7beb30c 100644 --- a/client/platform/desktop/backend/platforms/common.ts +++ b/client/platform/desktop/backend/platforms/common.ts @@ -1,31 +1,30 @@ /** * Common native implementations */ +import { AddressInfo } from 'net'; import npath from 'path'; import fs from 'fs-extra'; import { shell } from 'electron'; import mime from 'mime-types'; import moment from 'moment'; -import type { TrackData } from 'vue-media-annotator/track'; import { - DatasetType, MultiTrackRecord, Pipelines, SaveDetectionsArgs, FrameImage, + DatasetType, MultiTrackRecord, Pipelines, SaveDetectionsArgs, FrameImage, DatasetMetaMutable, } from 'viame-web-common/apispec'; import { - JsonMeta, Settings, websafeImageTypes, websafeVideoTypes, JsonMetaCurrentVersion, DesktopDataset, otherImageTypes, + websafeImageTypes, websafeVideoTypes, otherImageTypes, + JsonMeta, Settings, JsonMetaCurrentVersion, DesktopDataset, } from 'platform/desktop/constants'; import * as viameSerializers from 'platform/desktop/backend/serializers/viame'; import { makeMediaUrl } from '../server'; import { cleanString, makeid } from './utils'; -import { number } from 'yargs'; const ProjectsFolderName = 'DIVE_Projects'; -const TrainedPipelinesFolderName = 'DIVE_Trained_Pipelines'; +const JobsFolderName = 'DIVE_Jobs'; const AuxFolderName = 'auxiliary'; -const JobFolderName = 'job_runs'; -const JsonFileName = /^result(_.*)?\.json$/; +const JsonTrackFileName = /^result(_.*)?\.json$/; const JsonMetaFileName = 'meta.json'; const CsvFileName = /^.*\.csv$/; @@ -38,7 +37,7 @@ async function _findJsonTrackFile(basePath: string): Promise { const contents = await fs.readdir(basePath); const jsonFileCandidates: string[] = []; await Promise.all(contents.map(async (name) => { - if (JsonFileName.test(name)) { + if (JsonTrackFileName.test(name)) { const fullPath = npath.join(basePath, name); const statResult = await fs.stat(fullPath); if (statResult.isFile()) { @@ -47,7 +46,7 @@ async function _findJsonTrackFile(basePath: string): Promise { } })); if (jsonFileCandidates.length > 1) { - throw new Error(`too many matches for json annotation file in ${basePath}`); + throw new Error(`too many matches for json annotation file in ${basePath}. cannot determine correct choice. please verify only 1 json annotation file exists.`); } else if (jsonFileCandidates.length === 1) { return jsonFileCandidates[0]; } @@ -55,38 +54,38 @@ async function _findJsonTrackFile(basePath: string): Promise { } /** - * _getProjectDir returns filepaths to required members of a dataset project directory. - * - * REQUIRED members: meta.json, results*.json - * - * OPTIONAL members: aux/ will be created if none exists - * - * @param settings user settings - * @param datasetId dataset id string + * getProjectDir returns filepaths to required members of a dataset project directory. */ -async function _getProjectDir(settings: Settings, datasetId: string) { +function getProjectDir(settings: Settings, datasetId: string) { const basePath = npath.join(settings.dataPath, ProjectsFolderName, datasetId); - if (!fs.pathExistsSync(basePath)) { - throw new Error(`missing project directory ${basePath}`); - } - const auxDirAbsPath = npath.join(basePath, AuxFolderName); - fs.ensureDirSync(auxDirAbsPath); - const metaFileAbsPath = npath.join(basePath, JsonMetaFileName); - if (!fs.pathExists(metaFileAbsPath)) { - throw new Error(`missing metadata json file ${metaFileAbsPath}`); - } - - const trackFileAbsPath = await _findJsonTrackFile(basePath); - if (trackFileAbsPath === null) { - throw new Error(`missing track json file in ${basePath}`); - } - return { auxDirAbsPath, basePath, metaFileAbsPath, + }; +} + +/** + * REQUIRED members: meta.json, results*.json + * OPTIONAL members: aux/ will be created if none exists + */ +async function getValidatedProjectDir(settings: Settings, datasetId: string) { + const projectInfo = getProjectDir(settings, datasetId); + fs.ensureDirSync(projectInfo.auxDirAbsPath); + if (!fs.pathExistsSync(projectInfo.basePath)) { + throw new Error(`missing project directory ${projectInfo.basePath}`); + } + if (!fs.pathExists(projectInfo.metaFileAbsPath)) { + throw new Error(`missing metadata json file ${projectInfo.metaFileAbsPath}`); + } + const trackFileAbsPath = await _findJsonTrackFile(projectInfo.basePath); + if (trackFileAbsPath === null) { + throw new Error(`missing track json file in ${projectInfo.basePath}`); + } + return { + ...projectInfo, trackFileAbsPath, }; } @@ -95,7 +94,7 @@ async function _getProjectDir(settings: Settings, datasetId: string) { * _loadJsonMeta processes dataset information from json * @param metaPath a known, existing path */ -async function _loadJsonMeta(metaAbsPath: string): Promise { +async function loadMetadata(metaAbsPath: string): Promise { const rawBuffer = await fs.readFile(metaAbsPath, 'utf-8'); const metaJson = JSON.parse(rawBuffer); /* check if this file meets the current schema version */ @@ -127,10 +126,13 @@ async function _loadJsonTracks(tracksAbsPath: string): Promise * loadDataset load detections and meta from disk * @param settings user settings * @param datasetId user data folder name + * @param addr server address TODO REMOVE THIS */ -async function loadDataset(settings: Settings, datasetId: string): Promise { - const projectDirData = await _getProjectDir(settings, datasetId); - const projectMetaData = await _loadJsonMeta(projectDirData.metaFileAbsPath); +async function loadDataset( + settings: Settings, datasetId: string, addr: AddressInfo, +): Promise { + const projectDirData = await getValidatedProjectDir(settings, datasetId); + const projectMetaData = await loadMetadata(projectDirData.metaFileAbsPath); let videoUrl = ''; let imageData = [] as FrameImage[]; @@ -139,18 +141,16 @@ async function loadDataset(settings: Settings, datasetId: string): Promise ({ - url: makeMediaUrl(npath.join(projectMetaData.originalBasePath, filename)), + url: makeMediaUrl(npath.join(projectMetaData.originalBasePath, filename), addr), filename, })); } else { @@ -203,28 +203,21 @@ async function getPipelineList(settings: Settings): Promise { return ret; } -/** - * Create aux directory if none exists - * @param baseDir parent - */ -async function getAuxFolder(baseDir: string): Promise { - const auxFolderPath = npath.join(baseDir, AuxFolderName); - if (!fs.existsSync(auxFolderPath)) { - await fs.mkdir(auxFolderPath); - } - return auxFolderPath; -} - /** * Create `job_runs/{runfoldername}` folder, usually inside an aux folder * @param baseDir parent * @param pipeline name */ -async function createKwiverRunWorkingDir(datasetName: string, baseDir: string, pipeline: string) { - const jobFolderPath = npath.join(baseDir, JobFolderName); +async function createKwiverRunWorkingDir( + settings: Settings, jsonMetaList: JsonMeta[], pipeline: string, +) { + if (jsonMetaList.length === 0) { + throw new Error('At least 1 jsonMeta item must be provided'); + } + const jobFolderPath = npath.join(settings.dataPath, JobsFolderName); // eslint won't recognize \. as valid escape // eslint-disable-next-line no-useless-escape - const safeDatasetName = datasetName.replace(/[\.\s/]+/g, '_'); + const safeDatasetName = jsonMetaList[0].name.replace(/[\.\s/]+/g, '_'); const runFolderName = moment().format(`[${safeDatasetName}_${pipeline}]_MM-DD-yy_hh-mm-ss`); const runFolderPath = npath.join(jobFolderPath, runFolderName); if (!fs.existsSync(jobFolderPath)) { @@ -236,60 +229,84 @@ async function createKwiverRunWorkingDir(datasetName: string, baseDir: string, p /** * _saveSerialized save pre-serialized tracks to disk - * @param trackData json serialized track object */ async function _saveSerialized( + settings: Settings, datasetId: string, trackData: MultiTrackRecord, ) { - const time = moment().format('MM-DD-YYYY_HH-MM-SS'); + const time = moment().format('MM-DD-YYYY_hh-mm-ss'); const newFileName = `result_${time}.json`; - const base = await getDatasetBase(datasetId); - - const auxFolderPath = await getAuxFolder(base.basePath); + const projectInfo = getProjectDir(settings, datasetId); - /* Move old file if it exists */ - if (base.jsonFile) { + try { + const validatedInfo = await getValidatedProjectDir(settings, datasetId); await fs.move( - npath.join(base.basePath, base.jsonFile), - npath.join(auxFolderPath, base.jsonFile), + validatedInfo.trackFileAbsPath, + npath.join( + validatedInfo.auxDirAbsPath, + npath.basename(validatedInfo.trackFileAbsPath), + ), ); + } catch (err) { + // Some part of the project dir didn't exist } - const serialized = JSON.stringify(trackData); - - /* Save new file */ - await fs.writeFile(npath.join(base.basePath, newFileName), serialized); + await fs.writeFile(npath.join(projectInfo.basePath, newFileName), serialized); } /** * Save detections to json file in aux - * @param datasetId path - * @param args save args */ async function saveDetections(settings: Settings, datasetId: string, args: SaveDetectionsArgs) { /* Update existing track file */ - const projectDirInfo = await _getProjectDir(settings, datasetId); + const projectDirInfo = await getValidatedProjectDir(settings, datasetId); const existing = await _loadJsonTracks(projectDirInfo.trackFileAbsPath); args.delete.forEach((trackId) => delete existing[trackId.toString()]); args.upsert.forEach((track, trackId) => { existing[trackId.toString()] = track.serialize(); }); - return _saveSerialized(datasetId, existing); + return _saveSerialized(settings, datasetId, existing); +} + +/** + * _saveAsJson saves directly to disk + */ +async function _saveAsJson(absPath: string, data: unknown) { + const serialized = JSON.stringify(data, null, 2); + await fs.writeFile(absPath, serialized); +} + +async function saveMetadata(settings: Settings, datasetId: string, args: DatasetMetaMutable) { + const projectDirInfo = await getValidatedProjectDir(settings, datasetId); + const existing = await loadMetadata(projectDirInfo.metaFileAbsPath); + if (args.confidenceFilters) { + existing.confidenceFilters = args.confidenceFilters; + } + if (args.customTypeStyling) { + existing.customTypeStyling = args.customTypeStyling; + } + _saveAsJson(projectDirInfo.metaFileAbsPath, existing); } /** * processOtherAnnotationFiles imports data from external annotation formats + * given a list of candidate file paths. * - * Only VIAME CSV is currently supported. + * SUPPORTED FORMATS: + * VIAME CSV * * @param paths paths to possible input annotation files * @param datasetId dataset id path */ async function processOtherAnnotationFiles( - absPaths: string[], datasetId: string, -): Promise<{ fps?: number }> { + settings: Settings, + datasetId: string, + absPaths: string[], +): Promise<{ fps?: number; processedFiles: string[] }> { const fps = undefined; + const processedFiles = []; // which files were processed to generate the detections + for (let i = 0; i < absPaths.length; i += 1) { const path = absPaths[i]; if (!fs.existsSync(path)) { @@ -300,27 +317,28 @@ async function processOtherAnnotationFiles( // Attempt to process the file // eslint-disable-next-line no-await-in-loop const tracks = await viameSerializers.parseFile(path); - const data = {} as Record; + const data: MultiTrackRecord = {}; tracks.forEach((t) => { data[t.trackId.toString()] = t; }); // eslint-disable-next-line no-await-in-loop - await _saveSerialized(datasetId, data); + await _saveSerialized(settings, datasetId, data); + processedFiles.push(path); break; // Exit on first successful detection load } } - return { fps }; + return { fps, processedFiles }; } async function _initializeAppDataDir(settings: Settings) { await fs.ensureDir(settings.dataPath); await fs.ensureDir(npath.join(settings.dataPath, ProjectsFolderName)); - await fs.ensureDir(npath.join(settings.dataPath, TrainedPipelinesFolderName)); + await fs.ensureDir(npath.join(settings.dataPath, JobsFolderName)); } /** * Intialize a new project directory * @returns absolute path to new project dcirectory */ -async function _initializeProjectDir(settings: Settings, jsonMeta: JsonMeta): string { +async function _initializeProjectDir(settings: Settings, jsonMeta: JsonMeta): Promise { const projectDir = npath.join(settings.dataPath, ProjectsFolderName, jsonMeta.id); await _initializeAppDataDir(settings); await fs.ensureDir(projectDir); @@ -334,7 +352,7 @@ async function _initializeProjectDir(settings: Settings, jsonMeta: JsonMeta): st * @param path path to import dir/file * @returns datasetId */ -async function importMedia(settings: Settings, path: string): Promise { +async function importMedia(settings: Settings, path: string): Promise { let datasetType: DatasetType; const exists = fs.existsSync(path); @@ -353,21 +371,32 @@ async function importMedia(settings: Settings, path: string): Promise { const dsName = npath.parse(path).name; const dsId = `${cleanString(dsName).substr(0, 20)}_${makeid(10)}`; - let datasetFolderPath = path; + + const jsonMeta: JsonMeta = { + version: JsonMetaCurrentVersion, + type: datasetType, + id: dsId, + fps: 5, // TODO + originalBasePath: path, + originalVideoFile: '', + originalImageFiles: [], + transcodedVideoFile: '', // TODO: this is empty (see above) + transcodedImageFiles: [], // TODO: this is empty + name: dsName, + }; + + /* TODO: Look for an EXISTING meta.json file to override the above */ + if (datasetType === 'video') { // get parent folder, since videos reference a file directly - datasetFolderPath = npath.dirname(path); + jsonMeta.originalBasePath = npath.dirname(path); } - let imageFiles: string[] = []; - const transcodedImageFiles: string[] = []; // TODO: unused - let videoFile = ''; - const transcodedVideoFile = ''; // TODO: unused - const contents = await fs.readdir(datasetFolderPath); + const contents = await fs.readdir(jsonMeta.originalBasePath); /* Extract and validate media from import path */ - if (datasetType === 'video') { - videoFile = npath.basename(path); + if (jsonMeta.type === 'video') { + jsonMeta.originalVideoFile = npath.basename(path); const mimetype = mime.lookup(path); if (mimetype) { if (websafeImageTypes.includes(mimetype) || otherImageTypes.includes(mimetype)) { @@ -376,64 +405,69 @@ async function importMedia(settings: Settings, path: string): Promise { /* TODO: Kick off video inspection and maybe transcode */ } } else { - throw new Error(`Could not determine video MIME type for ${path}`); + throw new Error(`could not determine video MIME type for ${path}`); } } else if (datasetType === 'image-sequence') { - imageFiles = contents.filter((filename) => { - const abspath = npath.join(datasetFolderPath, filename); + jsonMeta.originalImageFiles = contents.filter((filename) => { + const abspath = npath.join(jsonMeta.originalBasePath, filename); const mimetype = mime.lookup(abspath); /* TODO: support transcoding of non-web-safe image types */ return !!(mimetype && websafeImageTypes.includes(mimetype)); }); + if (jsonMeta.originalImageFiles.length === 0) { + throw new Error(`no images found in ${path}`); + } } else { - throw new Error('Only video and image-sequence types are supported'); + throw new Error('only video and image-sequence types are supported'); } - - // TODO: parse FPS from CSV if it exists - - const jsonMeta: JsonMeta = { - version: JsonMetaCurrentVersion, - type: datasetType, - id: dsId, - fps: 5, // TODO - originalMediaAbsolutePath: path, - imageFiles, - name: dsName, - }; - const projectDirAbsPath = await _initializeProjectDir(settings, jsonMeta); + await _saveAsJson(npath.join(projectDirAbsPath, JsonMetaFileName), jsonMeta); + + let foundDetections = false; - /* Look for JSON track file */ - const trackFileAbsPath = await _findJsonTrackFile(datasetFolderPath); + /* Look for JSON track file as first priority */ + const trackFileAbsPath = await _findJsonTrackFile(jsonMeta.originalBasePath); if (trackFileAbsPath !== null) { /* Move the track file into the new project directory */ await fs.move( trackFileAbsPath, npath.join(projectDirAbsPath, npath.basename(trackFileAbsPath)), ); - /* Look for other supported annotation types */ - } else { + foundDetections = true; + } + /* Look for other types of annotation files as a second priority */ + if (!foundDetections) { const csvFileCandidates = contents .filter((v) => CsvFileName.test(v)) - .map((filename) => npath.join(datasetFolderPath, filename)); - const { fps } = await processOtherAnnotationFiles(csvFileCandidates, dsId); - + .map((filename) => npath.join(jsonMeta.originalBasePath, filename)); + const { fps, processedFiles } = await processOtherAnnotationFiles( + settings, dsId, csvFileCandidates, + ); + if (fps) jsonMeta.fps = fps; + foundDetections = processedFiles.length > 0; + } + /* Finally create an empty file as fallback */ + if (!foundDetections) { + await _saveSerialized(settings, dsId, {}); } + return jsonMeta; } - async function openLink(url: string) { shell.openExternal(url); } export default { - openLink, - getAuxFolder, createKwiverRunWorkingDir, getPipelineList, - loadDetections, + getProjectDir, + importMedia, + loadDataset, + loadMetadata, + openLink, + processOtherAnnotationFiles, saveDetections, - postprocess, + saveMetadata, }; diff --git a/client/platform/desktop/backend/platforms/linux.ts b/client/platform/desktop/backend/platforms/linux.ts index 1edbc3541..f0d262b91 100644 --- a/client/platform/desktop/backend/platforms/linux.ts +++ b/client/platform/desktop/backend/platforms/linux.ts @@ -10,7 +10,7 @@ import { Settings, SettingsCurrentVersion, DesktopJob, DesktopJobUpdate, RunPipeline, NvidiaSmiReply, -} from '../../constants'; +} from 'platform/desktop/constants'; import common from './common'; @@ -57,42 +57,41 @@ async function runPipeline( runPipelineArgs: RunPipeline, updater: (msg: DesktopJobUpdate) => void, ): Promise { - const { datasetId, pipelineName } = runPipelineArgs; + const { datasetId, pipeline } = runPipelineArgs; const isValid = await validateViamePath(settings); if (isValid !== true) { throw new Error(isValid); } const setupScriptPath = npath.join(settings.viamePath, 'setup_viame.sh'); - const pipelinePath = npath.join(settings.viamePath, 'configs/pipelines', pipelineName); - const datasetInfo = await common.getDatasetBase(datasetId); - const auxPath = await common.getAuxFolder(datasetInfo.basePath); - const jobWorkDir = await common.createKwiverRunWorkingDir( - datasetInfo.name, auxPath, pipelineName, - ); + const pipelinePath = npath.join(settings.viamePath, 'configs/pipelines', pipeline.pipe); + const projectInfo = await common.getProjectDir(settings, datasetId); + const meta = await common.loadMetadata(projectInfo.metaFileAbsPath); + const jobWorkDir = await common.createKwiverRunWorkingDir(settings, [meta], pipeline.name); const detectorOutput = npath.join(jobWorkDir, 'detector_output.csv'); const trackOutput = npath.join(jobWorkDir, 'track_output.csv'); const joblog = npath.join(jobWorkDir, 'runlog.txt'); let command: string[] = []; - if (datasetInfo.datasetType === 'video') { + if (meta.type === 'video') { + const videoAbsPath = npath.join(meta.originalBasePath, meta.originalVideoFile); command = [ `source ${setupScriptPath} &&`, 'kwiver runner', '-s input:video_reader:type=vidl_ffmpeg', `-p ${pipelinePath}`, - `-s input:video_filename=${datasetId}`, + `-s input:video_filename=${videoAbsPath}`, `-s detector_writer:file_name=${detectorOutput}`, `-s track_writer:file_name=${trackOutput}`, `| tee ${joblog}`, ]; - } else if (datasetInfo.datasetType === 'image-sequence') { + } else if (meta.type === 'image-sequence') { // Create frame image manifest const manifestFile = npath.join(jobWorkDir, 'image-manifest.txt'); // map image file names to absolute paths - const fileData = datasetInfo.imageFiles - .map((f) => npath.join(datasetInfo.basePath, f)) + const fileData = meta.originalImageFiles + .map((f) => npath.join(meta.originalBasePath, f)) .join('\n'); await fs.writeFile(manifestFile, fileData); command = [ @@ -115,7 +114,7 @@ async function runPipeline( key: `pipeline_${job.pid}_${jobWorkDir}`, jobType: 'pipeline', pid: job.pid, - pipelineName, + pipeline, workingDir: jobWorkDir, datasetIds: [datasetId], exitCode: job.exitCode, @@ -149,7 +148,9 @@ async function runPipeline( // eslint-disable-next-line no-console if (code === 0) { try { - await common.postprocess([trackOutput, detectorOutput], datasetId); + await common.processOtherAnnotationFiles( + settings, datasetId, [trackOutput, detectorOutput], + ); } catch (err) { console.error(err); } diff --git a/client/platform/desktop/backend/platforms/windows.ts b/client/platform/desktop/backend/platforms/windows.ts index f15d696cd..3f00ce1a3 100644 --- a/client/platform/desktop/backend/platforms/windows.ts +++ b/client/platform/desktop/backend/platforms/windows.ts @@ -11,7 +11,7 @@ import { Settings, SettingsCurrentVersion, DesktopJob, DesktopJobUpdate, RunPipeline, NvidiaSmiReply, -} from '../../constants'; +} from 'platform/desktop/constants'; import common from './common'; @@ -72,18 +72,18 @@ async function runPipeline( runPipelineArgs: RunPipeline, updater: (msg: DesktopJobUpdate) => void, ): Promise { - const { datasetId, pipelineName } = runPipelineArgs; + const { datasetId, pipeline } = runPipelineArgs; const isValid = await validateViamePath(settings); if (isValid !== true) { throw new Error(isValid); } const setupScriptPath = npath.join(settings.viamePath, 'setup_viame.bat'); - const pipelinePath = npath.join(settings.viamePath, 'configs/pipelines', pipelineName); - const datasetInfo = await common.getDatasetBase(datasetId); - const auxPath = await common.getAuxFolder(datasetInfo.basePath); + const pipelinePath = npath.join(settings.viamePath, 'configs/pipelines', pipeline.pipe); + const projectInfo = await common.getProjectDir(settings, datasetId); + const meta = await common.loadMetadata(projectInfo.metaFileAbsPath); const jobWorkDir = await common.createKwiverRunWorkingDir( - datasetInfo.name, auxPath, pipelineName, + settings, [meta], pipeline.name, ); const detectorOutput = npath.join(jobWorkDir, 'detector_output.csv'); @@ -93,7 +93,7 @@ async function runPipeline( const modifiedCommand = `"${setupScriptPath.replace(/\\/g, '\\')}"`; let command: string[] = []; - if (datasetInfo.datasetType === 'video') { + if (meta.type === 'video') { command = [ `${modifiedCommand} &&`, 'kwiver.exe runner', @@ -103,12 +103,12 @@ async function runPipeline( `-s detector_writer:file_name=${detectorOutput}`, `-s track_writer:file_name=${trackOutput}`, ]; - } else if (datasetInfo.datasetType === 'image-sequence') { + } else if (meta.type === 'image-sequence') { // Create frame image manifest const manifestFile = npath.join(jobWorkDir, 'image-manifest.txt'); // map image file names to absolute paths - const fileData = datasetInfo.imageFiles - .map((f) => npath.join(datasetInfo.basePath, f)) + const fileData = meta.originalImageFiles + .map((f) => npath.join(projectInfo.basePath, f)) .join('\n'); await fs.writeFile(manifestFile, fileData); command = [ @@ -130,7 +130,7 @@ async function runPipeline( key: `pipeline_${job.pid}_${jobWorkDir}`, jobType: 'pipeline', pid: job.pid, - pipelineName, + pipeline, workingDir: jobWorkDir, datasetIds: [datasetId], exitCode: job.exitCode, @@ -170,7 +170,9 @@ async function runPipeline( job.on('exit', async (code) => { if (code === 0) { try { - await common.postprocess([trackOutput, detectorOutput], datasetId); + await common.processOtherAnnotationFiles( + settings, datasetId, [trackOutput, detectorOutput], + ); } catch (err) { console.error(err); } @@ -186,9 +188,7 @@ async function runPipeline( return jobBase; } -function checkDefaultNvidiaSmi( - resolve: (value?: NvidiaSmiReply | PromiseLike | undefined) => void, -) { +function checkDefaultNvidiaSmi(resolve: (value: NvidiaSmiReply) => void) { const smi = spawn(`"${programFiles}\\NVIDIA Corporation\\NVSMI\\nvidia-smi.exe"`, ['-q', '-x'], { shell: true }); let result = ''; smi.stdout.on('data', (chunk) => { @@ -244,6 +244,7 @@ async function nvidiaSmi(): Promise { }); }); } + export default { DefaultSettings, validateViamePath, diff --git a/client/platform/desktop/backend/server.ts b/client/platform/desktop/backend/server.ts index 93f47d3f2..0bb3db9e0 100644 --- a/client/platform/desktop/backend/server.ts +++ b/client/platform/desktop/backend/server.ts @@ -37,7 +37,6 @@ const withErrorHandler = (handler: http.RequestListener): http.RequestListener = try { return handler(req, res); } catch (err) { - console.error('BAD'); return fail(res, 500, 'Server error'); } }; @@ -48,21 +47,6 @@ router.register('/api', withErrorHandler((req, res) => { res.end(); })); -router.register('/api/dataset', withErrorHandler((req, res) => { - const { url } = req; - if (!url) { - throw new Error('Impossible scenario, req.url was empty'); - } - const parsedurl = parser.parse(url, true); - let datasetId = parsedurl.query ? parsedurl.query.datasetId : undefined; - - if (datasetId === undefined || Array.isArray(datasetId)) { - return fail(res, 404, `Invalid dataset ID: ${datasetId}`); - } - - datasetId = decodeURI(datasetId); -})); - router.register('/api/media', withErrorHandler((req, res) => { const { url } = req; if (!url) { @@ -132,12 +116,14 @@ const server = http.createServer((req, res) => { /** * makeMediaUrl gets a URL for the given file path + * TODO: this will move to the backend, and should be called + * directly on the server */ -export function makeMediaUrl(filepath: string) { - const addr = server.address() as AddressInfo | null; - if (!addr) { - throw new Error('server has not initialized yet'); - } +export function makeMediaUrl(filepath: string, addr: AddressInfo): string { + // const addr = server.address() as AddressInfo | null; + // if (!addr) { + // throw new Error('server has not initialized yet'); + // } return `http://localhost:${addr.port}/api/media?path=${filepath}`; } diff --git a/client/platform/desktop/backend/state/settings.ts b/client/platform/desktop/backend/state/settings.ts new file mode 100644 index 000000000..c2e80e5c5 --- /dev/null +++ b/client/platform/desktop/backend/state/settings.ts @@ -0,0 +1,19 @@ +import type { Settings } from 'platform/desktop/constants'; + +let settings: Settings; + +function get(): Settings { + if (settings === undefined) { + throw new Error('Settings has not been initialized!'); + } + return settings; +} + +function set(s: Settings) { + settings = s; +} + +export default { + get, + set, +}; diff --git a/client/platform/desktop/components/Jobs.vue b/client/platform/desktop/components/Jobs.vue index ac8a2a3fe..44d954cdf 100644 --- a/client/platform/desktop/components/Jobs.vue +++ b/client/platform/desktop/components/Jobs.vue @@ -81,12 +81,12 @@ export default defineComponent({ - {{ job.datasets[0].name }} + {{ job.datasets[0].meta.name }} - [pipe] {{ job.job.pipelineName }} + [pipe] {{ job.job.pipeline.name }}
[pid] {{ job.job.pid }} -
[path] {{ job.datasets[0].basePath }} +
[path] {{ job.datasets[0].meta.originalBasePath }} diff --git a/client/platform/desktop/components/Recent.vue b/client/platform/desktop/components/Recent.vue index 1cdbcd959..f70a08d26 100644 --- a/client/platform/desktop/components/Recent.vue +++ b/client/platform/desktop/components/Recent.vue @@ -5,6 +5,7 @@ import { defineComponent } from '@vue/composition-api'; import { DatasetType } from 'viame-web-common/apispec'; import { openFromDisk } from '../api/main'; +import { importMedia } from '../store'; import { getRecents } from '../store/dataset'; import BrowserLink from './BrowserLink.vue'; import NavigationBar from './NavigationBar.vue'; @@ -19,9 +20,10 @@ export default defineComponent({ async function open(dstype: DatasetType) { const ret = await openFromDisk(dstype); if (!ret.canceled) { + const meta = await importMedia(ret.filePaths[0]); root.$router.push({ name: 'viewer', - params: { path: ret.filePaths[0] }, + params: { id: meta.id }, }); } } @@ -93,7 +95,7 @@ export default defineComponent({

@@ -101,15 +103,23 @@ export default defineComponent({ class="pr-2" color="primary lighten-2" > - {{ recent.ext ? 'mdi-file-video' : 'mdi-folder-open' }} + {{ + (recent.type === 'video') + ? 'mdi-file-video' + : (recent.originalImageFiles.length > 1) + ? 'mdi-image-multiple' + : 'mdi-image' + }} - {{ recent.base }} + {{ recent.name }} - {{ recent.dir }} + + {{ recent.originalBasePath }} +

diff --git a/client/platform/desktop/components/Settings.vue b/client/platform/desktop/components/Settings.vue index 40bae633c..235eaacf1 100644 --- a/client/platform/desktop/components/Settings.vue +++ b/client/platform/desktop/components/Settings.vue @@ -30,13 +30,13 @@ export default defineComponent({ smi.value = await nvidiaSmi(); }); - async function openPath() { + async function openPath(name: 'viamePath' | 'dataPath') { const result = await remote.dialog.showOpenDialog({ properties: ['openDirectory'], - defaultPath: localSettings.value.viamePath, + defaultPath: localSettings.value[name], }); if (!result.canceled) { - [localSettings.value.viamePath] = result.filePaths; + [localSettings.value[name]] = result.filePaths; } } @@ -86,15 +86,39 @@ export default defineComponent({ block color="primary" class="mb-6" - @click="openPath()" + @click="openPath('viamePath')" > - Open + Choose + + mdi-folder-open + + + + + + + + + + + Choose mdi-folder-open - diff --git a/client/platform/desktop/components/ViewerLoader.vue b/client/platform/desktop/components/ViewerLoader.vue index 629e28889..82cbcc9ff 100644 --- a/client/platform/desktop/components/ViewerLoader.vue +++ b/client/platform/desktop/components/ViewerLoader.vue @@ -14,20 +14,20 @@ export default defineComponent({ Viewer, }, props: { - path: { + id: { type: String, required: true, }, }, setup(props) { - const dataset = getDataset(props.path); + const dataset = getDataset(props.id); return { dataset }; }, }); diff --git a/client/platform/desktop/constants.ts b/client/platform/desktop/constants.ts index c80e69fc9..f1a0043af 100644 --- a/client/platform/desktop/constants.ts +++ b/client/platform/desktop/constants.ts @@ -1,5 +1,5 @@ import type { - DatasetMeta, DatasetMetaMutable, DatasetSchema, DatasetType, + DatasetMeta, DatasetMetaMutable, DatasetSchema, DatasetType, Pipe, } from 'viame-web-common/apispec'; export const websafeVideoTypes = [ @@ -17,7 +17,7 @@ export const websafeImageTypes = [ // 'image/webp', ]; -export const otherSupportedImageTypes = [ +export const otherImageTypes = [ 'image/avif', 'image/tiff', ]; @@ -35,9 +35,9 @@ export interface Settings { } /** - * JsonMetadata is a SUBSET of DatasetMeta contained within + * JsonMeta is a SUBSET of DatasetMeta contained within * the JsonFileSchema. The remaining parts of DatasetMeta must - * be calculated at load time. + * be generated at load time. */ export interface JsonMeta extends DatasetMetaMutable { // version used to manage schema migrations @@ -72,11 +72,15 @@ export interface JsonMeta extends DatasetMetaMutable { // ordered image filenames of transcoded images // relative to project path - transcodedImageFiles: string[]; + transcodedImageFiles?: string[]; + + // If the dataset required transcoding, specify the job + // key that ran transcoding + transcodingJobKey?: string; } export interface DesktopDataset extends DatasetSchema { - meta: JsonMeta & DatasetMeta; // JsonMeta satisfies DatasetMeta + meta: JsonMeta & DatasetMeta; } interface NvidiaSmiTextRecord { @@ -103,8 +107,8 @@ export interface DesktopJob { key: string; // jobType identify type of job jobType: 'pipeline' | 'training'; - // pipelineName of the pipe or job being run - pipelineName: string; + // pipe + pipeline: Pipe; // datasetIds of the involved datasets datasetIds: string[]; // pid of the process spawned @@ -126,5 +130,5 @@ export interface DesktopJobUpdate extends DesktopJob { export interface RunPipeline { datasetId: string; - pipelineName: string; + pipeline: Pipe; } diff --git a/client/platform/desktop/main.ts b/client/platform/desktop/main.ts index 4d4a94b2e..397903361 100644 --- a/client/platform/desktop/main.ts +++ b/client/platform/desktop/main.ts @@ -7,6 +7,7 @@ import vMousetrap from 'viame-web-common/vue-utilities/v-mousetrap'; import vuetify from './plugins/vuetify'; import router from './router'; +import { migrate } from './store'; import App from './App.vue'; Vue.config.productionTip = false; @@ -15,6 +16,7 @@ Vue.use(snackbarService(vuetify)); Vue.use(promptService(vuetify)); Vue.use(vMousetrap); +migrate(); new Vue({ vuetify, diff --git a/client/platform/desktop/router.ts b/client/platform/desktop/router.ts index 9a544aa4c..7c7b597e1 100644 --- a/client/platform/desktop/router.ts +++ b/client/platform/desktop/router.ts @@ -26,7 +26,7 @@ export default new Router({ component: Jobs, }, { - path: '/viewer/:path', + path: '/viewer/:id', name: 'viewer', component: ViewerLoader, props: true, diff --git a/client/platform/desktop/store/dataset.ts b/client/platform/desktop/store/dataset.ts index 1105bc90d..25af280fd 100644 --- a/client/platform/desktop/store/dataset.ts +++ b/client/platform/desktop/store/dataset.ts @@ -1,9 +1,7 @@ -import path from 'path'; - import Vue from 'vue'; -import { uniq } from 'lodash'; +import { uniqBy } from 'lodash'; import Install, { ref, computed } from '@vue/composition-api'; -import { DesktopDataset } from '../constants'; +import { DesktopDataset, JsonMeta } from '../constants'; const RecentsKey = 'desktop.recent'; @@ -24,31 +22,29 @@ function getDataset(id: string) { /** * Load recent datasets from localstorage */ -function getRecents(): path.ParsedPath[] { +function getRecents(): JsonMeta[] { const arr = window.localStorage.getItem(RecentsKey); - let returnVal = [] as path.ParsedPath[]; try { if (arr) { const maybeArr = JSON.parse(arr); if (maybeArr.length) { - returnVal = maybeArr.map((p: string) => path.parse(p)); + return maybeArr; } } } catch (err) { - return returnVal; + return []; } - return returnVal; + return []; } /** * Add ID to recent datasets * @param id dataset id path */ -function setRecents(id: string) { +function setRecents(meta: JsonMeta) { const recents = getRecents(); - recents.splice(0, 0, path.parse(id)); // verify that it's a valid path - let recentsStrings = recents.map((r) => path.join(r.dir, r.base)); - recentsStrings = uniq(recentsStrings); + recents.splice(0, 0, meta); // verify that it's a valid path + const recentsStrings = uniqBy(recents, ({ id }) => id); window.localStorage.setItem(RecentsKey, JSON.stringify(recentsStrings)); } @@ -60,11 +56,12 @@ function setRecents(id: string) { */ function setDataset(id: string, ds: DesktopDataset) { Vue.set(dsmap.value, id, ds); - setRecents(id); + setRecents(ds.meta); } export { getDataset, setDataset, getRecents, + RecentsKey, }; diff --git a/client/platform/desktop/store/index.ts b/client/platform/desktop/store/index.ts index 1fd1313f9..d8c5fc222 100644 --- a/client/platform/desktop/store/index.ts +++ b/client/platform/desktop/store/index.ts @@ -1,33 +1,65 @@ -import { Api } from 'viame-web-common/apispec'; -import * as api from '../api/main'; +import { + Api, DatasetMetaMutable, Pipe, SaveDetectionsArgs, +} from 'viame-web-common/apispec'; +import * as api from 'platform/desktop/api/main'; +/** + * TODO: Danger! + * These are "backend" methods that involve node.js. They should not be imported or + * used from client code. They should be refactored to run over REST or IPC + */ +import common from 'platform/desktop/backend/platforms/common'; import { settings } from './settings'; -import { setDataset, getDataset } from './dataset'; +import { + setDataset, getDataset, getRecents, RecentsKey, +} from './dataset'; import { getOrCreateHistory } from './jobs'; +/* Run forward migrations on any client-side data stores */ +export async function migrate() { + const recents = await getRecents(); + if (recents.length && typeof recents[0] === 'string') { + window.localStorage.setItem(RecentsKey, JSON.stringify([])); + } +} + +export async function importMedia(path: string) { + return common.importMedia(settings.value, path); +} + /** * Wrap API with hooks to use the store */ export default function wrap(): Api { - async function loadMetadata(datasetId: string) { - const ds = await api.loadMetadata(datasetId); - setDataset(datasetId, ds); - return ds.meta; + // TODO: see above + function saveDetections(datasetId: string, args: SaveDetectionsArgs) { + return common.saveDetections(settings.value, datasetId, args); } - async function getPipelineList() { - return api.getPipelineList(settings.value); + // TODO: see above + function saveMetadata(datasetId: string, args: DatasetMetaMutable) { + return common.saveMetadata(settings.value, datasetId, args); + } + + // TODO: see above + async function loadDataset(datasetId: string) { + const addrInfo = await api.mediaServerInfo(); + const ds = await common.loadDataset(settings.value, datasetId, addrInfo); + setDataset(datasetId, ds); + return ds; } - async function runPipeline(itemId: string, pipeline: string) { - const job = await api.runPipeline(itemId, pipeline, settings.value); + async function runPipeline(itemId: string, pipeline: Pipe) { + const job = await api.runPipeline(itemId, pipeline); const datasets = job.datasetIds.map(((id) => getDataset(id).value)); getOrCreateHistory(job, datasets); } return { ...api, - getPipelineList, + loadDataset, + saveDetections, + saveMetadata, runPipeline, }; } diff --git a/client/platform/web-girder/App.vue b/client/platform/web-girder/App.vue index 93374a475..6f08ab3f3 100644 --- a/client/platform/web-girder/App.vue +++ b/client/platform/web-girder/App.vue @@ -6,8 +6,8 @@