diff --git a/client/platform/desktop/backend/native/attributeProcessor.ts b/client/platform/desktop/backend/native/attributeProcessor.ts new file mode 100644 index 000000000..97880a74c --- /dev/null +++ b/client/platform/desktop/backend/native/attributeProcessor.ts @@ -0,0 +1,84 @@ +import { Attributes } from 'platform/desktop/constants'; +import { MultiTrackRecord } from 'viame-web-common/apispec'; +import { StringKeyObject, TrackData } from 'vue-media-annotator/track'; + + +/** + * Processes a list of tracks and returns a MultiTrackRecord and an attributes object to be used + * @param tracks list of tracks to process for the attributes + */ +function processTrackAttributes(tracks: TrackData[]): +{data: MultiTrackRecord; attributes: Attributes} { + const attributeObj: Attributes = {}; + const trackMap: MultiTrackRecord = {}; + const testVals: Record> = {}; + + function processAttributes(attributes: StringKeyObject, type: 'track' | 'detection') { + Object.entries(attributes).forEach(([key, val]) => { + const valstring = `${val}`; + if (attributeObj[`${type}_${key}`] === undefined) { + attributeObj[`${type}_${key}`] = { + belongs: type, + datatype: 'text', + name: key, + _id: `${type}_${key}`, + }; + testVals[`${type}_${key}`] = { }; + testVals[`${type}_${key}`][valstring] = 1; + } else if (attributeObj[`${type}_${key}`] && testVals[`${type}_${key}`]) { + if (testVals[`${type}_${key}`][valstring]) { + testVals[`${type}_${key}`][valstring] += 1; + } + } + }); + // Now we attempt to process the attributes to infer the type. + // Cascading based on the attempting to convert and values + Object.keys(attributeObj).forEach((attributeKey) => { + if (testVals[attributeKey]) { + let attributeType: ('number' | 'boolean' | 'text') = 'number'; + let lowCount = 1; + const values: string[] = []; + Object.entries(testVals[attributeKey]).forEach(([key, val]) => { + if (val <= lowCount) { + lowCount = val; + } + values.push(key); + if (attributeType === 'number' && Number.isNaN(parseFloat(key))) { + attributeType = 'boolean'; + } + if (attributeType === 'boolean' && key !== 'true' && key !== 'false') { + attributeType = 'text'; + } + }); + // If all items are used 2 or more times it has discrete set Values otherwise + if (lowCount >= 2 && attributeType.indexOf('text') !== -1) { + attributeObj[attributeKey].values = values; + } + // eslint-disable-next-line no-param-reassign + attributeObj[attributeKey].datatype = attributeType; + } + }); + } + + function processTrackforAttributes(track: TrackData) { + if (track.attributes) { + processAttributes(track.attributes, 'track'); + } + if (track.features) { + track.features.forEach((item) => { + if (item.attributes) { + processAttributes(track.attributes, 'detection'); + } + }); + } + } + + tracks.forEach((t) => { + trackMap[t.trackId.toString()] = t; + processTrackforAttributes(t); + }); + + return { data: trackMap, attributes: attributeObj }; +} + +export default processTrackAttributes; diff --git a/client/platform/desktop/backend/native/common.spec.ts b/client/platform/desktop/backend/native/common.spec.ts index 55798b558..fe0a9eff3 100644 --- a/client/platform/desktop/backend/native/common.spec.ts +++ b/client/platform/desktop/backend/native/common.spec.ts @@ -10,6 +10,7 @@ import type { DesktopJobUpdate, DesktopJobUpdater, JsonMeta, Settings, } from 'platform/desktop/constants'; +import { Attribute } from 'viame-web-common/apispec'; import * as common from './common'; const pipelines = { @@ -114,6 +115,12 @@ mockfs({ notanimage: '', 'notanimage.txt': '', }, + metaAttributesID: { + 'foo.png': '', + 'bar.png': '', + notanimage: '', + 'notanimage.txt': '', + }, videoSuccess: { 'video1.avi': '', 'video1.mp4': '', @@ -174,6 +181,40 @@ mockfs({ 'result_2.json': '', auxiliary: {}, }, + metaAttributesID: { + 'meta.json': JSON.stringify({ + version: 1, + id: 'metaAttributesID', + type: 'image-sequence', + fps: 5, + originalBasePath: '/home/user/media/metaAttributesID', + originalImageFiles: [ + 'foo.png', + 'bar.png', + ], + attributes: { + // eslint-disable-next-line @typescript-eslint/camelcase + track_attribute1: { + belongs: 'track', + datatype: 'text', + values: ['value1', 'value2', 'value3'], + name: 'attribute1', + _id: 'track_attribute1', + }, + // eslint-disable-next-line @typescript-eslint/camelcase + detection_attribute1: { + belongs: 'detection', + datatype: 'number', + name: 'attribute1', + _id: 'detection_attribute1', + }, + }, + }), + 'result_whatever.json': JSON.stringify({}), + auxiliary: {}, + + + }, }, }, }); @@ -280,6 +321,66 @@ describe('native.common', () => { expect(meta.transcodingJobKey).toBe('jobKey'); expect(meta.type).toBe('video'); }); + + + it('getAtributes', async () => { + const meta = await common.getAttributes(settings, 'metaAttributesID'); + //Should return an array of data items + expect(meta.length).toBe(2); + expect(meta[0].values).toEqual(['value1', 'value2', 'value3']); + expect(meta[1].datatype).toBe('number'); + }); + + it('addAttribute', async () => { + const templateAttribute: Attribute = { + name: 'newAttribute', datatype: 'boolean', belongs: 'track', _id: '', + }; + await common.setAttribute(settings, 'metaAttributesID', { + data: templateAttribute, + }); + //Should return an array of data items + const meta = await common.getAttributes(settings, 'metaAttributesID'); + const newAttribute = meta.find((item) => item.name === 'newAttribute'); + expect(meta.length).toBe(3); + expect(newAttribute).toEqual(templateAttribute); + }); + + it('updateAttribute', async () => { + const templateAttribute: Attribute = { + name: 'newAttributeName', datatype: 'boolean', belongs: 'track', _id: 'track_attribute1', + }; + await common.setAttribute(settings, 'metaAttributesID', { + data: templateAttribute, + }); + //Should return an array of data items + const meta = await common.getAttributes(settings, 'metaAttributesID'); + const updatedAttribute = meta.find((item) => item._id === 'track_attribute1'); + expect(meta.length).toBe(3); + expect(updatedAttribute).toEqual(templateAttribute); + }); + + it('deleteAttribute', async () => { + const deleteAttribute: Attribute = { + name: 'attribute1', datatype: 'text', belongs: 'track', _id: 'track_attribute1', + }; + await common.deleteAttribute(settings, 'metaAttributesID', { data: deleteAttribute }); + const meta = await common.getAttributes(settings, 'metaAttributesID'); + //Should return an array of data items + expect(meta.length).toBe(2); + expect(meta[0].datatype).toBe('number'); + }); + + it('initial attribute creation', async () => { + const templateAttribute: Attribute = { + name: 'newAttribute', datatype: 'boolean', belongs: 'track', _id: '', + }; + await common.setAttribute(settings, 'projectid1VideoGood', { + data: templateAttribute, + }); + const meta = await common.getAttributes(settings, 'projectid1VideoGood'); + expect(meta.length).toBe(1); + expect(meta[0]).toEqual(templateAttribute); + }); }); afterAll(() => { diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index f00eef60c..61cf6f2fb 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -10,14 +10,16 @@ import moment from 'moment'; import lockfile from 'proper-lockfile'; import { DatasetType, MultiTrackRecord, Pipelines, SaveDetectionsArgs, - FrameImage, DatasetMetaMutable, TrainingConfigs, + FrameImage, DatasetMetaMutable, TrainingConfigs, Attribute, } from 'viame-web-common/apispec'; import * as viameSerializers from 'platform/desktop/backend/serializers/viame'; import { websafeImageTypes, websafeVideoTypes, otherImageTypes, otherVideoTypes, - JsonMeta, Settings, JsonMetaCurrentVersion, DesktopMetadata, DesktopJobUpdater, ConvertMedia, + JsonMeta, Settings, JsonMetaCurrentVersion, DesktopMetadata, + DesktopJobUpdater, ConvertMedia, Attributes, } from 'platform/desktop/constants'; +import processTrackAttributes from './attributeProcessor'; import { cleanString, makeid } from './utils'; const ProjectsFolderName = 'DIVE_Projects'; @@ -321,10 +323,52 @@ async function saveMetadata(settings: Settings, datasetId: string, args: Dataset if (args.customTypeStyling) { existing.customTypeStyling = args.customTypeStyling; } + if (args.attributes) { + existing.attributes = args.attributes; + } await _saveAsJson(projectDirInfo.metaFileAbsPath, existing); await release(); } +async function getAttributes(settings: Settings, datasetId: string): + Promise { + const projectDirData = await getValidatedProjectDir(settings, datasetId); + const projectMetaData = await loadJsonMetadata(projectDirData.metaFileAbsPath); + if (projectMetaData.attributes) { + return Object.values(projectMetaData.attributes); + } + return []; +} + +async function setAttribute(settings: Settings, datasetId: string, { data }: + {data: Attribute }) { + const projectDirData = await getValidatedProjectDir(settings, datasetId); + const projectMetaData = await loadJsonMetadata(projectDirData.metaFileAbsPath); + if (!projectMetaData.attributes) { + projectMetaData.attributes = {}; + } + // Reassign _id based on name if it is a new item + if (data._id === '') { + // eslint-disable-next-line no-param-reassign + data._id = `${data.belongs}_${data.name}`; + } + projectMetaData.attributes[data._id] = data; + await saveMetadata(settings, datasetId, projectMetaData); +} + +async function deleteAttribute(settings: Settings, datasetId: string, { data }: + { data: Attribute }) { + const projectDirData = await getValidatedProjectDir(settings, datasetId); + const projectMetaData = await loadJsonMetadata(projectDirData.metaFileAbsPath); + if (!projectMetaData.attributes) { + return; + } + if (projectMetaData.attributes[data._id]) { + delete projectMetaData.attributes[data._id]; + } + await saveMetadata(settings, datasetId, projectMetaData); +} + /** * processOtherAnnotationFiles imports data from external annotation formats * given a list of candidate file paths. @@ -339,9 +383,10 @@ async function processOtherAnnotationFiles( settings: Settings, datasetId: string, absPaths: string[], -): Promise<{ fps?: number; processedFiles: string[] }> { +): Promise<{ fps?: number; processedFiles: string[]; attributes?: Attributes }> { const fps = undefined; const processedFiles = []; // which files were processed to generate the detections + let attributes: Attributes = {}; for (let i = 0; i < absPaths.length; i += 1) { const path = absPaths[i]; @@ -354,10 +399,10 @@ async function processOtherAnnotationFiles( try { // eslint-disable-next-line no-await-in-loop const tracks = await viameSerializers.parseFile(path); - const data: MultiTrackRecord = {}; - tracks.forEach((t) => { data[t.trackId.toString()] = t; }); + const processed = processTrackAttributes(tracks); + attributes = processed.attributes; // eslint-disable-next-line no-await-in-loop - await _saveSerialized(settings, datasetId, data, true); + await _saveSerialized(settings, datasetId, processed.data, true); processedFiles.push(path); break; // Exit on first successful detection load } catch (err) { @@ -366,7 +411,7 @@ async function processOtherAnnotationFiles( } } } - return { fps, processedFiles }; + return { fps, processedFiles, attributes }; } async function _initializeAppDataDir(settings: Settings) { @@ -521,7 +566,6 @@ async function importMedia( jsonMeta.transcodingJobKey = jobBase.key; } - await _saveAsJson(npath.join(projectDirAbsPath, JsonMetaFileName), jsonMeta); let foundDetections = false; @@ -533,6 +577,10 @@ async function importMedia( trackFileAbsPath, npath.join(projectDirAbsPath, npath.basename(trackFileAbsPath)), ); + //Load tracks to generate attributes + const tracks = await loadJsonTracks(trackFileAbsPath); + const { attributes } = processTrackAttributes(Object.values(tracks)); + if (attributes) jsonMeta.attributes = attributes; foundDetections = true; } /* Look for other types of annotation files as a second priority */ @@ -543,12 +591,16 @@ async function importMedia( if (csvFileCandidates.length > 1) { throw new Error(`too many CSV files found in ${jsonMeta.originalBasePath}, expected at most 1`); } - const { fps, processedFiles } = await processOtherAnnotationFiles( + const { fps, processedFiles, attributes } = await processOtherAnnotationFiles( settings, dsId, csvFileCandidates, ); if (fps) jsonMeta.fps = fps; + if (attributes) jsonMeta.attributes = attributes; foundDetections = processedFiles.length > 0; } + + await _saveAsJson(npath.join(projectDirAbsPath, JsonMetaFileName), jsonMeta); + /* Finally create an empty file as fallback */ if (!foundDetections) { await _saveSerialized(settings, dsId, {}, true); @@ -593,4 +645,7 @@ export { saveDetections, saveMetadata, completeConversion, + getAttributes, + setAttribute, + deleteAttribute, }; diff --git a/client/platform/desktop/backend/server.ts b/client/platform/desktop/backend/server.ts index f0d814c6f..180226ccd 100644 --- a/client/platform/desktop/backend/server.ts +++ b/client/platform/desktop/backend/server.ts @@ -8,7 +8,7 @@ import express from 'express'; import bodyparser from 'body-parser'; import rangeParser from 'range-parser'; import fs from 'fs-extra'; -import { SaveDetectionsArgs } from 'viame-web-common/apispec'; +import { Attribute, SaveDetectionsArgs } from 'viame-web-common/apispec'; import settings from './state/settings'; import * as common from './native/common'; @@ -64,6 +64,41 @@ apirouter.post('/dataset/:id/meta', async (req, res, next) => { } }); +/* LOAD Attributes */ +apirouter.get('/dataset/:id/attribute', async (req, res, next) => { + try { + const ds = await common.getAttributes(settings.get(), req.params.id); + res.json(ds); + } catch (err) { + err.status = 500; + next(err); + } +}); + +/* ADD/Update Attribute */ +apirouter.post('/dataset/:id/attribute', async (req, res, next) => { + try { + const args = req.body as {addNew?: boolean; data: Attribute }; + await common.setAttribute(settings.get(), req.params.id, args); + res.status(200).send('done'); + } catch (err) { + err.status = 500; + next(err); + } +}); + +/* Delete Attribute */ +apirouter.delete('/dataset/:id/attribute', async (req, res, next) => { + try { + const args = { data: req.body } as { data: Attribute }; + await common.deleteAttribute(settings.get(), req.params.id, args); + res.status(200).send('done'); + } catch (err) { + err.status = 500; + next(err); + } +}); + /* SAVE detections */ apirouter.post('/dataset/:id/detections', async (req, res, next) => { try { diff --git a/client/platform/desktop/constants.ts b/client/platform/desktop/constants.ts index 2dd266735..20ad9a4c0 100644 --- a/client/platform/desktop/constants.ts +++ b/client/platform/desktop/constants.ts @@ -1,4 +1,5 @@ import type { + Attribute, DatasetMeta, DatasetMetaMutable, DatasetType, Pipe, } from 'viame-web-common/apispec'; @@ -52,6 +53,8 @@ export interface Settings { dataPath: string; } +export type Attributes = Record; + /** * JsonMeta is a SUBSET of DatasetMeta contained within * the JsonFileSchema. The remaining parts of DatasetMeta must diff --git a/client/platform/desktop/frontend/api.ts b/client/platform/desktop/frontend/api.ts index f4da30adf..541579ee0 100644 --- a/client/platform/desktop/frontend/api.ts +++ b/client/platform/desktop/frontend/api.ts @@ -109,22 +109,21 @@ async function saveDetections(id: string, args: SaveDetectionsArgs) { return client.post(`dataset/${id}/detections`, args); } -/** - * Unimplemented sections of the API - */ - -async function getAttributes() { - return Promise.resolve([] as Attribute[]); +async function getAttributes(datasetId: string) { + const client = await getClient(); + const { data } = await client.get(`dataset/${datasetId}/attribute`); + return data; } -// eslint-disable-next-line @typescript-eslint/no-unused-vars -async function setAttribute({ addNew, data }: {addNew: boolean | undefined; data: Attribute}) { - return Promise.resolve(); +async function setAttribute(datasetId: string, { addNew, data }: + {addNew?: boolean; data: Attribute}) { + const client = await getClient(); + return client.post(`dataset/${datasetId}/attribute`, { addNew, data }); } -// eslint-disable-next-line @typescript-eslint/no-unused-vars -async function deleteAttribute(data: Attribute) { - return Promise.resolve([] as Attribute[]); +async function deleteAttribute(datasetId: string, data: Attribute) { + const client = await getClient(); + return client.delete(`dataset/${datasetId}/attribute`, { data }); } export { diff --git a/client/platform/web-girder/api/viame.service.ts b/client/platform/web-girder/api/viame.service.ts index 59cc58655..b79851d84 100644 --- a/client/platform/web-girder/api/viame.service.ts +++ b/client/platform/web-girder/api/viame.service.ts @@ -64,12 +64,13 @@ function deleteResources(resources: Array) { headers: { 'X-HTTP-Method-Override': 'DELETE' }, }); } - -async function getAttributes(): Promise { +// eslint-disable-next-line @typescript-eslint/no-unused-vars +async function getAttributes(datasetId = ''): Promise { const { data } = await girderRest.get('/viame/attribute'); return data as Attribute[]; } -function setAttribute({ addNew, data }: +// eslint-disable-next-line @typescript-eslint/no-unused-vars +function setAttribute(datasetId = '', { addNew, data }: {addNew: boolean | undefined; data: Attribute}): Promise { if (addNew) { return girderRest.post('/viame/attribute', data); @@ -79,8 +80,8 @@ function setAttribute({ addNew, data }: data, ); } - -function deleteAttribute(data: Attribute): Promise { +// eslint-disable-next-line @typescript-eslint/no-unused-vars +function deleteAttribute(datasetId = '', data: Attribute): Promise { return girderRest.delete( `/viame/attribute/${data._id}`, ); diff --git a/client/src/provides.ts b/client/src/provides.ts index 80d137a85..02fb24e76 100644 --- a/client/src/provides.ts +++ b/client/src/provides.ts @@ -17,6 +17,9 @@ import { RectBounds } from './utils'; const AllTypesSymbol = Symbol('allTypes'); type AllTypesType = Readonly>; +const DatasetIdSymbol = Symbol('datasetID'); +type DatasetIdType = Readonly>; + const UsedTypesSymbol = Symbol('usedTypes'); type UsedTypesType = Readonly>; @@ -158,6 +161,7 @@ function dummyHandler(handle: (name: string, args: unknown[]) => void): Handler */ export interface State { allTypes: AllTypesType; + datasetId: DatasetIdType; usedTypes: UsedTypesType; checkedTrackIds: CheckedTrackIdsType; checkedTypes: CheckedTypesType; @@ -187,6 +191,7 @@ function dummyState(): State { }; return { allTypes: ref([]), + datasetId: ref(''), usedTypes: ref([]), checkedTrackIds: ref([]), checkedTypes: ref([]), @@ -223,6 +228,7 @@ function dummyState(): State { */ function provideAnnotator(state: State, handler: Handler) { provide(AllTypesSymbol, state.allTypes); + provide(DatasetIdSymbol, state.datasetId); provide(UsedTypesSymbol, state.usedTypes); provide(CheckedTrackIdsSymbol, state.checkedTrackIds); provide(CheckedTypesSymbol, state.checkedTypes); @@ -255,6 +261,9 @@ function use(s: symbol) { function useAllTypes() { return use(AllTypesSymbol); } +function useDatasetId() { + return use(DatasetIdSymbol); +} function useUsedTypes() { return use(UsedTypesSymbol); } @@ -321,6 +330,7 @@ export { provideAnnotator, use, useAllTypes, + useDatasetId, useUsedTypes, useCheckedTrackIds, useCheckedTypes, diff --git a/client/viame-web-common/apispec.ts b/client/viame-web-common/apispec.ts index ba449e97d..3cfd9d88c 100644 --- a/client/viame-web-common/apispec.ts +++ b/client/viame-web-common/apispec.ts @@ -49,6 +49,7 @@ interface FrameImage { interface DatasetMetaMutable { customTypeStyling?: Record; confidenceFilters?: Record; + attributes?: Record; } interface DatasetMeta extends DatasetMetaMutable { @@ -63,11 +64,15 @@ interface DatasetMeta extends DatasetMetaMutable { interface Api { /** + * TODO: Modification to use loadMetadata as well as saving + * utilizing upsert/delete for the metaData. This requires having + * useAttributes to manage attributes locally and then save to backend * @deprecated soon attributes will come from loadMetadata() */ - getAttributes(): Promise; - setAttribute({ addNew, data }: {addNew: boolean | undefined; data: Attribute}): Promise; - deleteAttribute(data: Attribute): Promise; + getAttributes(datasetId: string): Promise; + setAttribute(datasetId: string, { addNew, data }: + {addNew?: boolean; data: Attribute}): Promise; + deleteAttribute(datasetId: string, data: Attribute): Promise; getPipelineList(): Promise; runPipeline(itemId: string, pipeline: Pipe): Promise; diff --git a/client/viame-web-common/components/AttributesSubsection.vue b/client/viame-web-common/components/AttributesSubsection.vue index eb81a839f..51a04ec6a 100644 --- a/client/viame-web-common/components/AttributesSubsection.vue +++ b/client/viame-web-common/components/AttributesSubsection.vue @@ -39,7 +39,7 @@ export default defineComponent({ const frameRef = useFrame(); const selectedTrackIdRef = useSelectedTrackId(); const trackMap = useTrackMap(); - const activeSettings = ref(false); + const activeSettings = ref(true); const selectedTrack = computed(() => { if (selectedTrackIdRef.value !== null) { diff --git a/client/viame-web-common/components/TrackDetailsPanel.vue b/client/viame-web-common/components/TrackDetailsPanel.vue index 3c8732d6f..7c4ab521a 100644 --- a/client/viame-web-common/components/TrackDetailsPanel.vue +++ b/client/viame-web-common/components/TrackDetailsPanel.vue @@ -15,6 +15,7 @@ import { useAllTypes, useHandler, useTrackMap, + useDatasetId, } from 'vue-media-annotator/provides'; import { getTrack } from 'vue-media-annotator/use/useTrackStore'; import TrackItem from 'vue-media-annotator/components/TrackItem.vue'; @@ -47,7 +48,6 @@ export default defineComponent({ type: Boolean, required: true, }, - }, setup(props) { const attributes = ref([] as Attribute[]); @@ -57,6 +57,7 @@ export default defineComponent({ const typeStylingRef = useTypeStyling(); const allTypesRef = useAllTypes(); const trackMap = useTrackMap(); + const datasetId = useDatasetId(); const { trackSelectNext, trackSplit, removeTrack } = useHandler(); //Edit/Set single value by clicking @@ -96,7 +97,7 @@ export default defineComponent({ async function closeEditor() { editingAttribute.value = null; editingError.value = null; - attributes.value = await getAttributes(); + attributes.value = await getAttributes(datasetId.value); } function addAttribute(type: 'Track' | 'Detection') { @@ -126,7 +127,7 @@ export default defineComponent({ } try { - await setAttribute(saveData); + await setAttribute(datasetId.value, saveData); } catch (err) { editingError.value = err.message; } @@ -137,7 +138,7 @@ export default defineComponent({ async function deleteAttributeHandler(data: Attribute) { editingError.value = null; try { - await deleteAttribute(data); + await deleteAttribute(datasetId.value, data); } catch (err) { editingError.value = err.message; } @@ -181,7 +182,7 @@ export default defineComponent({ }); onBeforeMount(async () => { - attributes.value = await getAttributes(); + attributes.value = await getAttributes(datasetId.value); }); return { diff --git a/client/viame-web-common/components/Viewer.vue b/client/viame-web-common/components/Viewer.vue index f3367ed31..e8e45cb9a 100644 --- a/client/viame-web-common/components/Viewer.vue +++ b/client/viame-web-common/components/Viewer.vue @@ -292,6 +292,7 @@ export default defineComponent({ provideAnnotator( { allTypes, + datasetId: ref(props.id), usedTypes, checkedTrackIds, checkedTypes,