Desktop CSV Serializer

client/README.md

@@ -20,23 +20,49 @@ yarn build:web
 # build vue-media-server library
 yarn build:lib
 
-# build electron
+# Electron
+yarn serve:electron
 yarn build:electron
 
-# lint
+# lint and test
 yarn lint
 yarn lint:templates
+yarn test
 
 # Local verification of all tests, linting, builds
 ./checkbuild.sh
+```
+
+### CLI Tools
+
+``` bash
+# Build
+yarn build:cli
+
+# Watch
+yarn dev:cli
+
+# Run in development mode
+yarn divecli --help
+
+# Parse VIAME CSV
+yarn divecli viame2json /path/to/viame.csv
+
+# Parse DIVE JSON
+yarn divecli json2viame /path/to/results.json /path/to/meta.json
 
-# Parser CLI tools
-yarn serialize viame /path/to/viame.csv
 # output to file, suppress yarn's stdout
-yarn --silent serialize viame /path/to/viame.csv > tracks.json
+yarn --silent divecli viame2json /path/to/viame.csv > tracks.json
 ```
 
-See [this issue](https://github.com/vuejs/vue-cli/issues/3065) for details on why our `yarn serve` command is weird.
+Configuration abnormalities:
+
+> **Note** tsconfig.cli.json is used to build the cli.  It's necessary to specify the exact files along the import path of the cli.js entrypoint, and if new files are added, they will need to be added manually.  Build errors should alert you to this.
+
+* `tsconfig.json`: `{ target: 'es2018' }` used because renderer/web uses babel but background does not, and [webpack doesn't support esnext](https://stackoverflow.com/questions/58813176/webpack-cant-compile-ts-3-7-optional-chaining-nullish-coalescing)
+* [acorn unexpected token webpack issue (unused, just useful)](https://github.com/webpack/webpack/issues/10227)
+* [Why our yarn serve is weird](https://github.com/vuejs/vue-cli/issues/3065)
+* [Typescript Absolute -> Relative Paths](https://github.com/microsoft/TypeScript/issues/15479)
 
 ## Publishing
 

client/checkbuild.sh

@@ -8,11 +8,14 @@ set -e # exit immediately on fail
 
 yarn install --frozen-lockfile
 
-yarn test &
+yarn test
+
 yarn lint &
-yarn lint:templates &
+yarn lint:templates
+
 yarn build:web &
 yarn build:electron &
 yarn build:lib &
+yarn build:cli &
 
 time wait 

client/package.json

@@ -13,11 +13,12 @@
     "build:web": "vue-cli-service build platform/web-girder/main.ts",
     "build:electron": "vue-cli-service electron:build",
     "build:lib": "rollup -c",
-    "build:cli": "tsc -p tsconfig.cli.json",
+    "build:cli": "tsc -b tsconfig.cli.json && tsc-alias -p tsconfig.cli.json",
+    "dev:cli": "tsc -w -p tsconfig.cli.json & tsc-alias -w -p tsconfig.cli.json",
+    "divecli": "node ./bin/platform/desktop/backend/serializers/cli.js",
     "lint": "vue-cli-service lint src/ viame-web-common/ platform/",
     "lint:templates": "vtc --workspace . --srcDir src/",
-    "test": "vue-cli-service test:unit src/ viame-web-common/ platform/",
-    "serialize": "ts-node --project tsconfig.cli.json platform/desktop/backend/serializers/cli.ts"
+    "test": "vue-cli-service test:unit src/ viame-web-common/ platform/"
   },
   "resolutions": {
     "@types/jest": "^25.2.3"
@@ -27,7 +28,7 @@
     "/lib/"
   ],
   "bin": {
-    "viamecli": "./bin/platform/desktop/backend/serializers/cli.js"
+    "divecli": "./bin/platform/desktop/backend/serializers/cli.js"
   },
   "main": "lib/index.js",
   "types": "lib/types/index.d.ts",
@@ -42,6 +43,7 @@
     "@vue/composition-api": "^1.0.0-beta.18",
     "axios": "^0.21.1",
     "core-js": "^3.6.4",
+    "csv-stringify": "^5.6.0",
     "d3": "^5.12.0",
     "geojs": "^0.20.0",
     "lodash": "^4.17.19",
@@ -71,7 +73,6 @@
     "@types/range-parser": "^1.2.3",
     "@types/request": "^2.48.5",
     "@types/resize-observer-browser": "^0.1.4",
-    "@types/xml2json": "^0.11.3",
     "@typescript-eslint/eslint-plugin": "^2.33.0",
     "@typescript-eslint/parser": "^2.33.0",
     "@vue/cli-plugin-babel": "~4.3.1",
@@ -112,7 +113,7 @@
     "sass": "^1.26.3",
     "sass-loader": "^8.0.2",
     "ts-jest": "^26.0.0",
-    "ts-node": "^9.0.0",
+    "tsc-alias": "^1.2.0",
     "typescript": "~3.8.3",
     "vue-cli-plugin-electron-builder": "^2.0.0-beta.5",
     "vue-cli-plugin-vuetify": "^2.0.5",
@@ -121,7 +122,6 @@
     "vue-type-check": "^1.0.0",
     "vuetify-loader": "^1.4.3",
     "xml-js": "^1.6.11",
-    "xml2json": "^0.12.0",
     "yargs": "^16.1.0"
   },
   "jest": {

client/platform/desktop/backend/ipcService.ts

@@ -6,7 +6,7 @@ import { DesktopJobUpdate, RunPipeline, Settings } from 'platform/desktop/consta
 
 import linux from './native/linux';
 import win32 from './native/windows';
-import common from './native/common';
+import * as common from './native/common';
 import settings from './state/settings';
 import { listen } from './server';
 

client/platform/desktop/backend/native/common.spec.ts

@@ -6,7 +6,7 @@ import { Console } from 'console';
 
 import type { JsonMeta, Settings } from 'platform/desktop/constants';
 
-import common from './common';
+import * as common from './common';
 
 const pipelines = {
   'classify_detections_svm.pipe': '',

client/platform/desktop/backend/native/common.ts

@@ -9,12 +9,12 @@ import moment from 'moment';
 import {
   DatasetType, MultiTrackRecord, Pipelines, SaveDetectionsArgs, FrameImage, DatasetMetaMutable,
 } from 'viame-web-common/apispec';
+import * as viameSerializers from 'platform/desktop/backend/serializers/viame';
 
 import {
   websafeImageTypes, websafeVideoTypes, otherImageTypes,
   JsonMeta, Settings, JsonMetaCurrentVersion, DesktopMetadata,
 } from 'platform/desktop/constants';
-import * as viameSerializers from 'platform/desktop/backend/serializers/viame';
 
 import { cleanString, makeid } from './utils';
 
@@ -107,10 +107,10 @@ async function loadJsonMetadata(metaAbsPath: string): Promise<JsonMeta> {
 }
 
 /**
- * _loadJsonTracks load from file
+ * loadJsonTracks load from file
  * @param tracksPath a known, existing path
  */
-async function _loadJsonTracks(tracksAbsPath: string): Promise<MultiTrackRecord> {
+async function loadJsonTracks(tracksAbsPath: string): Promise<MultiTrackRecord> {
   const rawBuffer = await fs.readFile(tracksAbsPath, 'utf-8');
   const annotationData = JSON.parse(rawBuffer) as MultiTrackRecord;
   // TODO: somehow verify the schema of this file
@@ -160,7 +160,7 @@ async function loadMetadata(
 
 async function loadDetections(settings: Settings, datasetId: string) {
   const projectDirData = await getValidatedProjectDir(settings, datasetId);
-  return _loadJsonTracks(projectDirData.trackFileAbsPath);
+  return loadJsonTracks(projectDirData.trackFileAbsPath);
 }
 
 /**
@@ -257,7 +257,7 @@ async function _saveSerialized(
 async function saveDetections(settings: Settings, datasetId: string, args: SaveDetectionsArgs) {
   /* Update existing track file */
   const projectDirInfo = await getValidatedProjectDir(settings, datasetId);
-  const existing = await _loadJsonTracks(projectDirInfo.trackFileAbsPath);
+  const existing = await loadJsonTracks(projectDirInfo.trackFileAbsPath);
   args.delete.forEach((trackId) => delete existing[trackId.toString()]);
   args.upsert.forEach((track) => {
     existing[track.trackId.toString()] = track;
@@ -466,14 +466,15 @@ async function openLink(url: string) {
   shell.openExternal(url);
 }
 
-export default {
+export {
   createKwiverRunWorkingDir,
   getPipelineList,
   getProjectDir,
   getValidatedProjectDir,
   importMedia,
   loadMetadata,
   loadJsonMetadata,
+  loadJsonTracks,
   loadDetections,
   openLink,
   processOtherAnnotationFiles,

client/platform/desktop/backend/native/linux.ts

@@ -13,7 +13,7 @@ import {
   NvidiaSmiReply,
 } from 'platform/desktop/constants';
 
-import common from './common';
+import * as common from './common';
 
 const DefaultSettings: Settings = {
   // The current settings schema config

client/platform/desktop/backend/native/windows.ts

@@ -13,7 +13,7 @@ import {
   NvidiaSmiReply,
 } from 'platform/desktop/constants';
 
-import common from './common';
+import * as common from './common';
 
 const DefaultSettings: Settings = {
   // The current settings schema config

client/platform/desktop/backend/serializers/cli.ts

@@ -6,25 +6,57 @@
  * See README.md for usage
  */
 
+import { Writable } from 'stream';
+import { stdout } from 'process';
 import yargs from 'yargs';
 
-import { parseFile } from './viame';
+import { loadJsonTracks, loadJsonMetadata } from 'platform/desktop/backend/native/common';
+import { parseFile, serialize } from './viame';
+
+// https://stackoverflow.com/questions/21491567/how-to-implement-a-writable-stream
+function echoStream() {
+  return new Writable({
+    write(chunk, encoding, next) {
+      stdout.write(chunk.toString());
+      next();
+    },
+  });
+}
 
 async function parseViameFile(file: string) {
   const tracks = await parseFile(file);
-  // eslint-disable-next-line no-console
-  console.log(JSON.stringify(tracks));
+  stdout.write(JSON.stringify(tracks));
+}
+
+async function parseJsonFile(filepath: string, metapath: string) {
+  await Promise.all([
+    loadJsonTracks(filepath),
+    loadJsonMetadata(metapath),
+  ]).then(([input, meta]) => serialize(echoStream(), input, meta));
 }
 
 const { argv } = yargs
-  .command('viame [file]', 'Parse VIAME CSV', (y) => {
+  .command('viame2json [file]', 'Convert VIAME CSV to JSON', (y) => {
+    y.positional('file', {
+      description: 'The file to parse',
+      type: 'string',
+    }).demandOption('file');
+  })
+  .command('json2viame [file] [meta]', 'Convert JSON to VIAME CSV', (y) => {
     y.positional('file', {
       description: 'The file to parse',
       type: 'string',
-    });
+    }).demandOption('file');
+    y.positional('meta', {
+      description: 'The metadata to parse',
+      type: 'string',
+    }).demandOption('meta');
   })
   .help();
 
-if (argv._.includes('viame')) {
+if (argv._.includes('viame2json')) {
   parseViameFile(argv.file as string);
 }
+if (argv._.includes('json2viame')) {
+  parseJsonFile(argv.file as string, argv.meta as string);
+}