From 64fe690c4478ef2ed2ccd41366d5d1b5194208b7 Mon Sep 17 00:00:00 2001 From: BryonLewis Date: Thu, 28 Jan 2021 12:00:16 -0500 Subject: [PATCH 1/7] initial for local training pipelines --- .../platform/desktop/backend/native/common.ts | 84 ++++++++++++++++++- .../platform/desktop/backend/native/viame.ts | 17 +++- 2 files changed, 98 insertions(+), 3 deletions(-) diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index f00eef60c..71205c1bf 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -16,12 +16,15 @@ import * as viameSerializers from 'platform/desktop/backend/serializers/viame'; import { websafeImageTypes, websafeVideoTypes, otherImageTypes, otherVideoTypes, - JsonMeta, Settings, JsonMetaCurrentVersion, DesktopMetadata, DesktopJobUpdater, ConvertMedia, + JsonMeta, Settings, JsonMetaCurrentVersion, DesktopMetadata, DesktopJobUpdater, + ConvertMedia, RunTraining, } from 'platform/desktop/constants'; import { cleanString, makeid } from './utils'; const ProjectsFolderName = 'DIVE_Projects'; const JobsFolderName = 'DIVE_Jobs'; +const PipelinesFolderName = 'DIVE_Pipelines'; + const AuxFolderName = 'auxiliary'; const JsonTrackFileName = /^result(_.*)?\.json$/; @@ -212,6 +215,50 @@ async function getPipelineList(settings: Settings): Promise { }; } }); + + // Now lets add to it the trained pipelines by recursively looking in the dir + const allowedTrainedPatterns = /^detector.+|^tracker.+|^generate.+|^trained_detector\.zip|^trained_tracker\.zip|^trained_generate\.zip/; + const trainedPipelinePath = npath.join(settings.dataPath, PipelinesFolderName); + const trainedExists = await fs.pathExists(trainedPipelinePath); + if (!trainedExists) return ret; + const trainedPipeFolders = await fs.readdir(trainedPipelinePath); + console.log(trainedPipeFolders); + await Promise.all(trainedPipeFolders.map(async (item) => { + const pipeFolder = npath.join(trainedPipelinePath, item); + const pipeFolderExists = await fs.pathExists(pipeFolder); + if (!pipeFolderExists) return false; + let pipesInFolder = await fs.readdir(pipeFolder); + console.log('pipesInFolder'); + console.log(pipesInFolder); + pipesInFolder = pipesInFolder.filter( + (p: string) => p.match(allowedTrainedPatterns) && !p.match(disallowedPatterns), + ); + console.log('pipesInFolder'); + console.log(pipesInFolder); + if (pipesInFolder.length >= 2) { + const pipeName = pipesInFolder.find((pipe) => pipe && pipe.indexOf('.pipe') !== -1); + console.log('pipeName'); + console.log(pipeName); + if (pipeName) { + const pipeInfo = { + name: item, + type: 'trained', + pipe: npath.join(pipeFolder, pipeName), + }; + if ('trained' in ret) { + ret.trained.pipes.push(pipeInfo); + } else { + ret.trained = { + pipes: [pipeInfo], + description: 'trained pipes', + }; + } + } + } + return true; + })); + console.log('Pipeline Return Value'); + console.log(ret); return ret; } @@ -368,6 +415,40 @@ async function processOtherAnnotationFiles( } return { fps, processedFiles }; } +/** + * Need to take the trained pipeline if it exists and place it in the DIVE_Pipelines folder + */ +async function processTrainedPipeline(settings: Settings, args: RunTraining, workingDir: string) { + //Look for trained_detector.zip and detector.pipe and move them to DIVE_Pipelines folder + const allowedPatterns = /^detector_.+|^tracker_.+|^generate_.+|^trained_detector\.zip|^trained_tracker\.zip|^trained_generate\.zip/; + const trainedDir = npath.join(workingDir, '/category_models'); + const exists = await fs.pathExists(trainedDir); + if (!exists) return {}; + let pipes = await fs.readdir(trainedDir); + console.log(pipes); + pipes = pipes.filter((p) => p.match(allowedPatterns)); + + if (!pipes.length) { + throw new Error('Could not located trained files'); + } + const baseFolder = npath.join(settings.dataPath, PipelinesFolderName); + if (!fs.existsSync(baseFolder)) { + await fs.mkdir(baseFolder); + } + + const folderName = npath.join(baseFolder, args.pipelineName); + if (!fs.existsSync(baseFolder)) { + await fs.mkdir(baseFolder); + } + + //Move detector and model to the new folder + pipes.forEach((item) => { + const abspath = npath.join(trainedDir, item); + const destpath = npath.join(folderName, item); + fs.move(abspath, destpath, { overwrite: true }); + }); + return false; +} async function _initializeAppDataDir(settings: Settings) { await fs.ensureDir(settings.dataPath); @@ -593,4 +674,5 @@ export { saveDetections, saveMetadata, completeConversion, + processTrainedPipeline, }; diff --git a/client/platform/desktop/backend/native/viame.ts b/client/platform/desktop/backend/native/viame.ts index 84a1b9bf2..a4ee36d41 100644 --- a/client/platform/desktop/backend/native/viame.ts +++ b/client/platform/desktop/backend/native/viame.ts @@ -46,7 +46,10 @@ async function runPipeline( throw new Error(isValid); } - const pipelinePath = npath.join(settings.viamePath, PipelineRelativeDir, pipeline.pipe); + let pipelinePath = npath.join(settings.viamePath, PipelineRelativeDir, pipeline.pipe); + if (runPipelineArgs.pipeline.type === 'trained') { + pipelinePath = pipeline.pipe; + } const projectInfo = await common.getValidatedProjectDir(settings, datasetId); const meta = await common.loadJsonMetadata(projectInfo.metaFileAbsPath); const jobWorkDir = await common.createKwiverRunWorkingDir(settings, [meta], pipeline.name); @@ -218,6 +221,7 @@ async function train( `--config "${configFilePath}"`, '--no-query', '--no-adv-prints', + '-s detector_trainer:ocv_windowed:trainer:netharn:timeout=100', ]; const job = spawn(command.join(' '), { @@ -249,7 +253,16 @@ async function train( job.stdout.on('data', jobFileEchoMiddleware(jobBase, updater, joblog)); job.stderr.on('data', jobFileEchoMiddleware(jobBase, updater, joblog)); - job.on('exit', (code) => { + job.on('exit', async (code) => { + if (code === 0) { + try { + await common.processTrainedPipeline( + settings, runTrainingArgs, jobWorkDir, + ); + } catch (err) { + console.error(err); + } + } updater({ ...jobBase, body: [''], From df7e55614c0fecb36be90695dd233bd281e75baf Mon Sep 17 00:00:00 2001 From: BryonLewis Date: Sun, 31 Jan 2021 10:42:38 -0500 Subject: [PATCH 2/7] Adding in name check --- .../frontend/components/MultiTrainingMenu.vue | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/client/platform/desktop/frontend/components/MultiTrainingMenu.vue b/client/platform/desktop/frontend/components/MultiTrainingMenu.vue index aa7b83499..7b8e916d5 100644 --- a/client/platform/desktop/frontend/components/MultiTrainingMenu.vue +++ b/client/platform/desktop/frontend/components/MultiTrainingMenu.vue @@ -2,15 +2,36 @@ import type { DataTableHeader } from 'vuetify'; import { - computed, defineComponent, onBeforeMount, set, del, reactive, + computed, defineComponent, onBeforeMount, set, del, reactive, ref, } from '@vue/composition-api'; -import { DatasetMeta, TrainingConfigs, useApi } from 'viame-web-common/apispec'; +import { + DatasetMeta, Pipelines, TrainingConfigs, useApi, +} from 'viame-web-common/apispec'; + import { datasets } from '../store/dataset'; export default defineComponent({ setup(_, { root }) { const { getTrainingConfigurations, runTraining } = useApi(); + const { getPipelineList } = useApi(); + const unsortedPipelines = ref({} as Pipelines); + onBeforeMount(async () => { + unsortedPipelines.value = await getPipelineList(); + }); + + const trainedPipelines = computed(() => { + const list: string[] = []; + Object.entries(unsortedPipelines.value).forEach(([, category]) => { + category.pipes.forEach((pipe) => { + if (pipe.type === 'trained') { + list.push(pipe.name); + } + }); + }); + return list; + }); + const data = reactive({ stagedItems: {} as Record, trainingOutputName: '', @@ -90,11 +111,16 @@ export default defineComponent({ } } + const nameRules = [ + (val: string) => (!trainedPipelines.value.includes(val) || 'A Trained pipeline with that name already exists'), + ]; + return { data, toggleStaged, isReadyToTrain, runTrainingOnFolder, + nameRules, available: { items: availableItems, headers: headersTmpl.concat( @@ -138,9 +164,9 @@ export default defineComponent({ From f6ce4c7e4837f8d4325685f7e261d91ba0e586e1 Mon Sep 17 00:00:00 2001 From: BryonLewis Date: Sun, 31 Jan 2021 16:49:26 -0500 Subject: [PATCH 3/7] adding in arguments --- client/platform/desktop/backend/native/common.ts | 12 +----------- client/platform/desktop/backend/native/viame.ts | 2 +- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index 71205c1bf..a7389e50d 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -222,23 +222,16 @@ async function getPipelineList(settings: Settings): Promise { const trainedExists = await fs.pathExists(trainedPipelinePath); if (!trainedExists) return ret; const trainedPipeFolders = await fs.readdir(trainedPipelinePath); - console.log(trainedPipeFolders); await Promise.all(trainedPipeFolders.map(async (item) => { const pipeFolder = npath.join(trainedPipelinePath, item); const pipeFolderExists = await fs.pathExists(pipeFolder); if (!pipeFolderExists) return false; let pipesInFolder = await fs.readdir(pipeFolder); - console.log('pipesInFolder'); - console.log(pipesInFolder); pipesInFolder = pipesInFolder.filter( (p: string) => p.match(allowedTrainedPatterns) && !p.match(disallowedPatterns), ); - console.log('pipesInFolder'); - console.log(pipesInFolder); if (pipesInFolder.length >= 2) { const pipeName = pipesInFolder.find((pipe) => pipe && pipe.indexOf('.pipe') !== -1); - console.log('pipeName'); - console.log(pipeName); if (pipeName) { const pipeInfo = { name: item, @@ -257,8 +250,6 @@ async function getPipelineList(settings: Settings): Promise { } return true; })); - console.log('Pipeline Return Value'); - console.log(ret); return ret; } @@ -420,12 +411,11 @@ async function processOtherAnnotationFiles( */ async function processTrainedPipeline(settings: Settings, args: RunTraining, workingDir: string) { //Look for trained_detector.zip and detector.pipe and move them to DIVE_Pipelines folder - const allowedPatterns = /^detector_.+|^tracker_.+|^generate_.+|^trained_detector\.zip|^trained_tracker\.zip|^trained_generate\.zip/; + const allowedPatterns = /^detector.+|^tracker.+|^generate.+|^trained_detector\.zip|^trained_tracker\.zip|^trained_generate\.zip/; const trainedDir = npath.join(workingDir, '/category_models'); const exists = await fs.pathExists(trainedDir); if (!exists) return {}; let pipes = await fs.readdir(trainedDir); - console.log(pipes); pipes = pipes.filter((p) => p.match(allowedPatterns)); if (!pipes.length) { diff --git a/client/platform/desktop/backend/native/viame.ts b/client/platform/desktop/backend/native/viame.ts index a4ee36d41..431b5f7c9 100644 --- a/client/platform/desktop/backend/native/viame.ts +++ b/client/platform/desktop/backend/native/viame.ts @@ -221,7 +221,7 @@ async function train( `--config "${configFilePath}"`, '--no-query', '--no-adv-prints', - '-s detector_trainer:ocv_windowed:trainer:netharn:timeout=100', + '--no-embedded-pipe', ]; const job = spawn(command.join(' '), { From e0e57f6e78c7695a5fd2bc517bd245b8854023c3 Mon Sep 17 00:00:00 2001 From: BryonLewis Date: Sun, 31 Jan 2021 17:24:45 -0500 Subject: [PATCH 4/7] Adding some better error detection --- client/platform/desktop/backend/native/common.ts | 14 ++++++++------ client/platform/desktop/backend/native/viame.ts | 12 ++++++++++-- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index a7389e50d..9b9b1c335 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -411,15 +411,17 @@ async function processOtherAnnotationFiles( */ async function processTrainedPipeline(settings: Settings, args: RunTraining, workingDir: string) { //Look for trained_detector.zip and detector.pipe and move them to DIVE_Pipelines folder - const allowedPatterns = /^detector.+|^tracker.+|^generate.+|^trained_detector\.zip|^trained_tracker\.zip|^trained_generate\.zip/; + const allowedPatterns = /^detector.+|^tracker.+|^generate.+/; const trainedDir = npath.join(workingDir, '/category_models'); const exists = await fs.pathExists(trainedDir); - if (!exists) return {}; - let pipes = await fs.readdir(trainedDir); - pipes = pipes.filter((p) => p.match(allowedPatterns)); + if (!exists) { + throw new Error(`Path: ${trainedDir} does not exist`); + } + const folderContents = await fs.readdir(trainedDir); + const pipes = folderContents.filter((p) => p.match(allowedPatterns)); if (!pipes.length) { - throw new Error('Could not located trained files'); + throw new Error(`Could not located trained pipe file inside of ${trainedDir}`); } const baseFolder = npath.join(settings.dataPath, PipelinesFolderName); if (!fs.existsSync(baseFolder)) { @@ -432,7 +434,7 @@ async function processTrainedPipeline(settings: Settings, args: RunTraining, wor } //Move detector and model to the new folder - pipes.forEach((item) => { + folderContents.forEach((item) => { const abspath = npath.join(trainedDir, item); const destpath = npath.join(folderName, item); fs.move(abspath, destpath, { overwrite: true }); diff --git a/client/platform/desktop/backend/native/viame.ts b/client/platform/desktop/backend/native/viame.ts index 431b5f7c9..5f2b897f5 100644 --- a/client/platform/desktop/backend/native/viame.ts +++ b/client/platform/desktop/backend/native/viame.ts @@ -222,6 +222,7 @@ async function train( '--no-query', '--no-adv-prints', '--no-embedded-pipe', + '-s detector_trainer:ocv_windowed:trainer:netharn:timeout=100', ]; const job = spawn(command.join(' '), { @@ -254,6 +255,8 @@ async function train( job.stdout.on('data', jobFileEchoMiddleware(jobBase, updater, joblog)); job.stderr.on('data', jobFileEchoMiddleware(jobBase, updater, joblog)); job.on('exit', async (code) => { + let exitCode = code; + const bodyText = ['']; if (code === 0) { try { await common.processTrainedPipeline( @@ -261,12 +264,17 @@ async function train( ); } catch (err) { console.error(err); + exitCode = 1; + bodyText.unshift(err.toString('utf-8')); + fs.appendFile(joblog, bodyText[0], (error) => { + if (error) throw error; + }); } } updater({ ...jobBase, - body: [''], - exitCode: code, + body: bodyText, + exitCode, endTime: new Date(), }); }); From f50b331c49fe513448dca02da426637606e565ea Mon Sep 17 00:00:00 2001 From: BryonLewis Date: Mon, 1 Feb 2021 09:10:55 -0500 Subject: [PATCH 5/7] removing timelimit --- client/platform/desktop/backend/native/viame.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/client/platform/desktop/backend/native/viame.ts b/client/platform/desktop/backend/native/viame.ts index 5f2b897f5..a1e8279c1 100644 --- a/client/platform/desktop/backend/native/viame.ts +++ b/client/platform/desktop/backend/native/viame.ts @@ -222,7 +222,6 @@ async function train( '--no-query', '--no-adv-prints', '--no-embedded-pipe', - '-s detector_trainer:ocv_windowed:trainer:netharn:timeout=100', ]; const job = spawn(command.join(' '), { From 260c79bfc2ecf2faeb65357237c52265879bf169 Mon Sep 17 00:00:00 2001 From: BryonLewis Date: Mon, 1 Feb 2021 09:59:46 -0500 Subject: [PATCH 6/7] adding in testing --- .../desktop/backend/native/common.spec.ts | 67 ++++++++++++++++++- .../platform/desktop/backend/native/common.ts | 13 ++-- 2 files changed, 71 insertions(+), 9 deletions(-) diff --git a/client/platform/desktop/backend/native/common.spec.ts b/client/platform/desktop/backend/native/common.spec.ts index 55798b558..a187a8851 100644 --- a/client/platform/desktop/backend/native/common.spec.ts +++ b/client/platform/desktop/backend/native/common.spec.ts @@ -7,7 +7,7 @@ import { Console } from 'console'; import type { ConversionArgs, DesktopJob, - DesktopJobUpdate, DesktopJobUpdater, JsonMeta, Settings, + DesktopJobUpdate, DesktopJobUpdater, JsonMeta, RunTraining, Settings, } from 'platform/desktop/constants'; import * as common from './common'; @@ -129,6 +129,25 @@ mockfs({ '/home/user/viamedata': { // eslint-disable-next-line @typescript-eslint/camelcase DIVE_Jobs: { + goodTrainingJob: { + // eslint-disable-next-line @typescript-eslint/camelcase + category_models: { + 'detector.pipe': '', + 'trained_detector.zip': '', + }, + }, + badTrainingJob: { + missingModelFolder: {}, + }, + missingPipeTrainingJob: { + // eslint-disable-next-line @typescript-eslint/camelcase + category_models: { + 'trained_detector.zip': '', + }, + }, + }, + // eslint-disable-next-line @typescript-eslint/camelcase + DIVE_Pipelines: { /* Empty */ }, // eslint-disable-next-line @typescript-eslint/camelcase @@ -187,7 +206,7 @@ describe('native.common', () => { expect(pipes.detector.pipes).toHaveLength(4); expect(pipes.tracker.pipes).toHaveLength(5); expect(pipes.generate.pipes).toHaveLength(4); - expect(pipes.training).toBeUndefined(); + expect(pipes.trained).toBeUndefined(); }); it('getValidatedProjectDir loads correct project directory', async () => { @@ -280,6 +299,50 @@ describe('native.common', () => { expect(meta.transcodingJobKey).toBe('jobKey'); expect(meta.type).toBe('video'); }); + + it('processing good Trained Pipeline folder', async () => { + const trainingArgs: RunTraining = { + datasetIds: ['randomID'], + pipelineName: 'trainedPipelineName', + trainingConfig: 'trainingConfig', + }; + const contents = await common.processTrainedPipeline(settings, trainingArgs, '/home/user/viamedata/DIVE_Jobs/goodTrainingJob/'); + expect(contents).toEqual(['detector.pipe', 'trained_detector.zip']); + //Data should be moved out of the current folder + const sourceFolder = fs.readdirSync('/home/user/viamedata/DIVE_Jobs/goodTrainingJob/category_models'); + expect(sourceFolder.length).toBe(0); + //Folders hould be created for new pipeline + const pipelineFolder = '/home/user/viamedata/DIVE_Pipelines/trainedPipelineName'; + const exists = fs.existsSync(pipelineFolder); + expect(exists).toBe(true); + const folderContents = fs.readdirSync(pipelineFolder); + expect(folderContents.length).toBe(2); + }); + + it('processing bad Trained Pipeline folders', async () => { + const trainingArgs: RunTraining = { + datasetIds: ['randomID'], + pipelineName: 'trainedBadPipelineName', + trainingConfig: 'trainingConfig', + }; + expect(common.processTrainedPipeline(settings, trainingArgs, '/home/user/viamedata/DIVE_Jobs/badTrainingJob/')).rejects.toThrow( + 'Path: /home/user/viamedata/Dive_Jobs/badTrainingJob/category_models does not exist', + ); + expect(common.processTrainedPipeline(settings, trainingArgs, '/home/user/viamedata/DIVE_Jobs/missingPipeTrainingJob/')).rejects.toThrow( + 'Could not located trained pipe file inside of /home/user/viamedata/Dive_Jobs/missingPipeTrainingJob/category_models', + ); + }); + + it('getPipelineList lists pipelines with Trained pipelines', async () => { + const exists = await fs.pathExists(settings.viamePath); + expect(exists).toBe(true); + const pipes = await common.getPipelineList(settings); + expect(pipes).toBeTruthy(); + expect(pipes.detector.pipes).toHaveLength(4); + expect(pipes.tracker.pipes).toHaveLength(5); + expect(pipes.generate.pipes).toHaveLength(4); + expect(pipes.trained.pipes).toHaveLength(1); + }); }); afterAll(() => { diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index 9b9b1c335..00fc54823 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -429,17 +429,16 @@ async function processTrainedPipeline(settings: Settings, args: RunTraining, wor } const folderName = npath.join(baseFolder, args.pipelineName); - if (!fs.existsSync(baseFolder)) { - await fs.mkdir(baseFolder); + if (!fs.existsSync(folderName)) { + await fs.mkdir(folderName); } - //Move detector and model to the new folder - folderContents.forEach((item) => { + await Promise.all(folderContents.map(async (item) => { const abspath = npath.join(trainedDir, item); const destpath = npath.join(folderName, item); - fs.move(abspath, destpath, { overwrite: true }); - }); - return false; + await fs.move(abspath, destpath, { overwrite: true }); + })); + return folderContents; } async function _initializeAppDataDir(settings: Settings) { From 935e8bd98e1a9cb5053bba5834b7b9452779d204 Mon Sep 17 00:00:00 2001 From: BryonLewis Date: Mon, 1 Feb 2021 13:37:04 -0500 Subject: [PATCH 7/7] updated computed property --- .../frontend/components/MultiTrainingMenu.vue | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/client/platform/desktop/frontend/components/MultiTrainingMenu.vue b/client/platform/desktop/frontend/components/MultiTrainingMenu.vue index 7b8e916d5..ad682b02b 100644 --- a/client/platform/desktop/frontend/components/MultiTrainingMenu.vue +++ b/client/platform/desktop/frontend/components/MultiTrainingMenu.vue @@ -21,17 +21,16 @@ export default defineComponent({ }); const trainedPipelines = computed(() => { - const list: string[] = []; - Object.entries(unsortedPipelines.value).forEach(([, category]) => { - category.pipes.forEach((pipe) => { - if (pipe.type === 'trained') { - list.push(pipe.name); - } - }); - }); - return list; + if (unsortedPipelines.value.trained) { + return unsortedPipelines.value.trained.pipes.map((item) => item.name); + } + return []; }); + const nameRules = [ + (val: string) => (!trainedPipelines.value.includes(val) || 'A Trained pipeline with that name already exists'), + ]; + const data = reactive({ stagedItems: {} as Record, trainingOutputName: '', @@ -111,10 +110,6 @@ export default defineComponent({ } } - const nameRules = [ - (val: string) => (!trainedPipelines.value.includes(val) || 'A Trained pipeline with that name already exists'), - ]; - return { data, toggleStaged,