diff --git a/client/platform/desktop/backend/ipcService.ts b/client/platform/desktop/backend/ipcService.ts index 7fdfd56e4..99182c109 100644 --- a/client/platform/desktop/backend/ipcService.ts +++ b/client/platform/desktop/backend/ipcService.ts @@ -58,7 +58,7 @@ export default function register() { return defaults; }); - ipcMain.handle('import-media', async (event, path: string) => { + ipcMain.handle('import-media', async (event, { path }: { path: string }) => { const updater = (update: DesktopJobUpdate) => { event.sender.send('job-update', update); }; diff --git a/client/platform/desktop/backend/native/linux.ts b/client/platform/desktop/backend/native/linux.ts index fc204f01d..90a63b3c1 100644 --- a/client/platform/desktop/backend/native/linux.ts +++ b/client/platform/desktop/backend/native/linux.ts @@ -34,12 +34,34 @@ const ViameLinuxConstants = { kwiverExe: 'kwiver', shell: '/bin/bash', ffmpeg: { + // Ready indicates that the proper ffmpeg settings have been learned from the system. + ready: false, initialization: '', // command to initialize path: '', // location of the ffmpeg executable - encoding: '', //encoding mode used + // Default video args + videoArgs: [ + '-c:v libx264', + '-preset slow', + '-crf 26', + '-c:a copy', + /** + * TODO: Upgrade to ffmpeg 4, use `round` instead of `ceil` + * 3.4 is part of 18.04LTS, so we should support it + * + * References: + * https://github.com/Kitware/dive/pull/602 (Anamorphic Video Support) + * https://video.stackexchange.com/questions/20871/how-do-i-convert-anamorphic-hdv-video-to-normal-h-264-video-with-ffmpeg-how-to + */ + '-vf "scale=ceil(iw*sar/2)*2:ceil(ih/2)*2,setsar=1"', + ].join(' '), }, }; +const ViameBundledFFMPEGVideoArgs = [ + '-c:v h264', + '-c: a copy', + '-vf "scale=ceil(iw*sar/2)*2:ceil(ih/2)*2,setsar=1"', +].join(' '); async function validateViamePath(settings: Settings): Promise { const setupScriptPath = npath.join(settings.viamePath, ViameLinuxConstants.setup); @@ -127,7 +149,7 @@ async function nvidiaSmi(): Promise { * Linux version is more complicated for multiple VIAME versions and local ffmpeg */ async function ffmpegCommand(settings: Settings) { - if (ViameLinuxConstants.ffmpeg.path !== '' && ViameLinuxConstants.ffmpeg.encoding !== '') { + if (ViameLinuxConstants.ffmpeg.ready) { return; } const setupScriptPath = npath.join(settings.viamePath, ViameLinuxConstants.setup); @@ -142,7 +164,7 @@ async function ffmpegCommand(settings: Settings) { if (viameffmpeg.output.includes('libx264')) { ViameLinuxConstants.ffmpeg.initialization = `source ${setupScriptPath} &&`; ViameLinuxConstants.ffmpeg.path = `"${settings.viamePath}/bin/ffmpeg"`; - ViameLinuxConstants.ffmpeg.encoding = '-c:v libx264 -preset slow -crf 26 -c:a copy'; + ViameLinuxConstants.ffmpeg.ready = true; return; } } @@ -156,7 +178,7 @@ async function ffmpegCommand(settings: Settings) { if (localffmpeg.output.includes('libx264')) { ViameLinuxConstants.ffmpeg.initialization = ''; ViameLinuxConstants.ffmpeg.path = 'ffmpeg'; - ViameLinuxConstants.ffmpeg.encoding = '-c:v libx264 -preset slow -crf 26 -c:a copy'; + ViameLinuxConstants.ffmpeg.ready = true; return; } } @@ -168,7 +190,8 @@ async function ffmpegCommand(settings: Settings) { if (ffmpegViameExists) { ViameLinuxConstants.ffmpeg.initialization = `source ${setupScriptPath} &&`; ViameLinuxConstants.ffmpeg.path = `"${settings.viamePath}/bin/ffmpeg"`; - ViameLinuxConstants.ffmpeg.encoding = '-c:v h264 -c:a copy'; + ViameLinuxConstants.ffmpeg.videoArgs = ViameBundledFFMPEGVideoArgs; + ViameLinuxConstants.ffmpeg.ready = true; return; } //We make it down here we have no way to convert the video file diff --git a/client/platform/desktop/backend/native/viame.ts b/client/platform/desktop/backend/native/viame.ts index 8e48e15d4..9ad1cf5f4 100644 --- a/client/platform/desktop/backend/native/viame.ts +++ b/client/platform/desktop/backend/native/viame.ts @@ -13,15 +13,15 @@ import * as common from './common'; import { cleanString, jobFileEchoMiddleware, spawnResult } from './utils'; const PipelineRelativeDir = 'configs/pipelines'; - +const DiveJobManifestName = 'dive_job_manifest.json'; interface FFmpegSettings { initialization: string; path: string; - encoding: string; + videoArgs: string; } -interface ViameConstants { +export interface ViameConstants { setupScriptAbs: string; // abs path setup comman trainingExe: string; // name of training binary on PATH kwiverExe: string; // name of kwiver binary on PATH @@ -60,7 +60,10 @@ async function runPipeline( let command: string[] = []; if (meta.type === 'video') { - const videoAbsPath = npath.join(meta.originalBasePath, meta.originalVideoFile); + let videoAbsPath = npath.join(meta.originalBasePath, meta.originalVideoFile); + if (meta.transcodedVideoFile) { + videoAbsPath = npath.join(projectInfo.basePath, meta.transcodedVideoFile); + } command = [ `${viameConstants.setupScriptAbs} &&`, `"${viameConstants.kwiverExe}" runner`, @@ -106,7 +109,7 @@ async function runPipeline( startTime: new Date(), }; - fs.writeFile(npath.join(jobWorkDir, 'dive_job_manifest.json'), JSON.stringify(jobBase)); + fs.writeFile(npath.join(jobWorkDir, DiveJobManifestName), JSON.stringify(jobBase)); updater({ ...jobBase, @@ -244,7 +247,7 @@ async function train( startTime: new Date(), }; - fs.writeFile(npath.join(jobWorkDir, 'dive_job_manifest.json'), JSON.stringify(jobBase)); + fs.writeFile(npath.join(jobWorkDir, DiveJobManifestName), JSON.stringify(jobBase)); updater({ ...jobBase, @@ -287,14 +290,14 @@ async function checkMedia( const ffprobePath = `${viameConstants.ffmpeg.path.replace('ffmpeg', 'ffprobe')}`; const command = [ `${viameConstants.ffmpeg.initialization}`, - `${ffprobePath}`, + `"${ffprobePath}"`, '-print_format', 'json', '-v', 'quiet', '-show_format', '-show_streams', - file, + `"${file}"`, ]; const result = await spawnResult(command.join(' '), viameConstants.shell); if (result.error || result.output === null) { @@ -303,7 +306,9 @@ async function checkMedia( const returnText = result.output; const ffprobeJSON: FFProbeResults = JSON.parse(returnText); if (ffprobeJSON && ffprobeJSON.streams) { - const websafe = ffprobeJSON.streams.filter((el) => el.codec_name === 'h264' && el.codec_type === 'video'); + const websafe = ffprobeJSON.streams + .filter((el) => el.codec_name === 'h264' && el.codec_type === 'video') + .filter((el) => el.sample_aspect_ratio === '1:1'); return !!websafe.length; } @@ -322,7 +327,13 @@ async function convertMedia(settings: Settings, const joblog = npath.join(jobWorkDir, 'runlog.txt'); const commands = []; if (args.meta.type === 'video' && args.mediaList[0]) { - commands.push(`${viameConstants.ffmpeg.initialization} ${viameConstants.ffmpeg.path} -i "${args.mediaList[0][0]}" ${viameConstants.ffmpeg.encoding} "${args.mediaList[0][1]}"`); + commands.push([ + viameConstants.ffmpeg.initialization, + viameConstants.ffmpeg.path, + `-i "${args.mediaList[0][0]}"`, + viameConstants.ffmpeg.videoArgs, + `"${args.mediaList[0][1]}"`, + ].join(' ')); } else if (args.meta.type === 'image-sequence' && imageIndex < args.mediaList.length) { commands.push(`${viameConstants.ffmpeg.initialization} ${viameConstants.ffmpeg.path} -i "${args.mediaList[imageIndex][0]}" "${args.mediaList[imageIndex][1]}"`); } @@ -345,13 +356,20 @@ async function convertMedia(settings: Settings, startTime: new Date(), }; + fs.writeFile(npath.join(jobWorkDir, DiveJobManifestName), JSON.stringify(jobBase)); + job.stdout.on('data', jobFileEchoMiddleware(jobBase, updater, joblog)); job.stderr.on('data', jobFileEchoMiddleware(jobBase, updater, joblog)); - job.on('exit', async (code) => { if (code !== 0) { console.error('Error with running conversion'); + updater({ + ...jobBase, + body: [''], + exitCode: code, + endTime: new Date(), + }); } else if (args.meta.type === 'video' || (args.meta.type === 'image-sequence' && imageIndex === args.mediaList.length - 1)) { common.completeConversion(settings, args.meta.id, jobKey); updater({ diff --git a/client/platform/desktop/backend/native/windows.ts b/client/platform/desktop/backend/native/windows.ts index 2772e0743..7311a2f92 100644 --- a/client/platform/desktop/backend/native/windows.ts +++ b/client/platform/desktop/backend/native/windows.ts @@ -31,9 +31,18 @@ const ViameWindowsConstants = { kwiverExe: 'kwiver.exe', shell: true, ffmpeg: { + ready: false, initialization: '', // command to initialize path: '', // location of the ffmpeg executable - encoding: '', //encoding mode used + // Default video args + videoArgs: [ + '-c:v libx264', + '-preset slow', + '-crf 26', + '-c:a copy', + // https://video.stackexchange.com/questions/20871/how-do-i-convert-anamorphic-hdv-video-to-normal-h-264-video-with-ffmpeg-how-to + '-vf "scale=ceil(iw*sar/2)*2:ceil(ih/2)*2,setsar=1"', + ].join(' '), }, }; @@ -162,7 +171,7 @@ async function nvidiaSmi(): Promise { * one time per launch configuration for ffmpeg and ffprobe */ async function ffmpegCommand(settings: Settings) { - if (ViameWindowsConstants.ffmpeg.path !== '' && ViameWindowsConstants.ffmpeg.encoding !== '') { + if (ViameWindowsConstants.ffmpeg.ready) { return; } const setupScriptPath = npath.join(settings.viamePath, ViameWindowsConstants.setup); @@ -178,7 +187,7 @@ async function ffmpegCommand(settings: Settings) { if (ffmpegOutput.includes('libx264')) { ViameWindowsConstants.ffmpeg.initialization = `"${setupScriptPath}" >NUL &&`; ViameWindowsConstants.ffmpeg.path = `"${settings.viamePath}/bin/ffmpeg.exe"`; - ViameWindowsConstants.ffmpeg.encoding = '-c:v libx264 -preset slow -crf 26 -c:a copy'; + ViameWindowsConstants.ffmpeg.ready = true; return; } } diff --git a/client/platform/desktop/background.ts b/client/platform/desktop/background.ts index 38d9187cf..ea3922d91 100644 --- a/client/platform/desktop/background.ts +++ b/client/platform/desktop/background.ts @@ -27,7 +27,7 @@ function cleanup() { app.quit(); } -function createWindow() { +async function createWindow() { const size = screen.getPrimaryDisplay().workAreaSize; // Create the browser window. win = new BrowserWindow({ @@ -44,9 +44,20 @@ function createWindow() { }, }); + listen((server) => { + let address = server.address(); + let port = 0; + if (typeof address === 'object' && address !== null) { + port = address.port || 0; + address = address.address || ''; + } + console.error(`Server listening on ${address}:${port}`); + }); + ipcListen(); + if (process.env.IS_ELECTRON) { // Load the url of the dev server if in development mode - win.loadURL(process.env.WEBPACK_DEV_SERVER_URL as string); + await win.loadURL(process.env.WEBPACK_DEV_SERVER_URL as string); if (!process.env.IS_TEST) win.webContents.openDevTools(); } else { createProtocol('app'); @@ -57,17 +68,6 @@ function createWindow() { win.on('closed', () => { win = null; }); - - listen((server) => { - let address = server.address(); - let port = 0; - if (typeof address === 'object' && address !== null) { - port = address.port || 0; - address = address.address || ''; - } - console.error(`Server listening on ${address}:${port}`); - }); - ipcListen(); } // Quit when all windows are closed. diff --git a/client/platform/desktop/constants.ts b/client/platform/desktop/constants.ts index 0c6a48f87..21431e6ad 100644 --- a/client/platform/desktop/constants.ts +++ b/client/platform/desktop/constants.ts @@ -9,9 +9,19 @@ export const websafeVideoTypes = [ ]; export const otherVideoTypes = [ - 'video/quicktime', + /* avi */ + 'vide/avi', + 'video/msvideo', 'video/x-msvideo', 'video/x-ms-wmv', + /* mov */ + 'video/quicktime', + /* mpeg */ + 'video/mpeg', + 'video/x-mpeg', + 'video/x-mpeq2a', + /* ogg */ + 'video/ogg', ]; export const fileVideoTypes = [ @@ -20,6 +30,10 @@ export const fileVideoTypes = [ 'avi', 'mov', 'wmv', + 'mpg', + 'mpeg', + 'mp2', + 'ogg', ]; export const websafeImageTypes = [ @@ -183,6 +197,7 @@ export interface FFProbeResults { streams?: [{ codec_type?: string; codec_name?: string; + sample_aspect_ratio?: string; }]; } diff --git a/client/platform/desktop/frontend/api.ts b/client/platform/desktop/frontend/api.ts index cbe126a96..872448b18 100644 --- a/client/platform/desktop/frontend/api.ts +++ b/client/platform/desktop/frontend/api.ts @@ -23,6 +23,7 @@ async function openFromDisk(datasetType: DatasetType) { if (datasetType === 'video') { filters = [ { name: 'Videos', extensions: fileVideoTypes }, + { name: 'All Files', extensions: ['*'] }, ]; } const results = await remote.dialog.showOpenDialog({ @@ -72,7 +73,7 @@ async function runTraining( } async function importMedia(path: string): Promise { - const data: JsonMeta = await ipcRenderer.invoke('import-media', path); + const data: JsonMeta = await ipcRenderer.invoke('import-media', { path }); return data; } diff --git a/server/dive_server/viame_detection.py b/server/dive_server/viame_detection.py index 4c00b3e6b..f86971939 100644 --- a/server/dive_server/viame_detection.py +++ b/server/dive_server/viame_detection.py @@ -46,7 +46,13 @@ def _get_clip_meta(self, folder): { 'folderId': folder['_id'], 'meta.codec': 'h264', - 'meta.source_video': {'$exists': False}, + 'meta.source_video': { + '$in': [ + # In a previous version, source_video was unset + None, + False, + ] + }, } ) if item: diff --git a/server/dive_tasks/tasks.py b/server/dive_tasks/tasks.py index 2207dcd75..444d8d4ef 100644 --- a/server/dive_tasks/tasks.py +++ b/server/dive_tasks/tasks.py @@ -13,7 +13,7 @@ from GPUtil import getGPUs from dive_tasks.utils import ( - get_source_video_filename, + get_video_filename, organize_folder_for_training, read_and_close_process_outputs, ) @@ -95,7 +95,7 @@ def run_pipeline(self: Task, params: PipelineJob): if input_type == 'video': # filter files for source video file - source_video = get_source_video_filename(input_folder, self.girder_client) + source_video = get_video_filename(input_folder, self.girder_client) # Preserving default behavior incase new stuff fails if source_video is None: raise Exception( @@ -252,7 +252,7 @@ def train_pipeline( ) # We point to file if is a video if source_folder.get("meta", {}).get("type") == "video": - video_file = get_source_video_filename(source_folder["_id"], gc) + video_file = get_video_filename(source_folder["_id"], gc) if video_file is None: raise Exception( 'Error finding valid video file in folder: {}'.format( @@ -397,6 +397,9 @@ def convert_video(self: Task, path, folderId, auxiliaryFolderId, itemId): "26", "-c:a", "copy", + # see native/ code for a discussion of this option + "-vf", + "scale=ceil(iw*sar/2)*2:ceil(ih/2)*2,setsar=1", output_path, ], stdout=process_log_file, @@ -415,7 +418,14 @@ def convert_video(self: Task, path, folderId, auxiliaryFolderId, itemId): if process.returncode == 0: manager.updateStatus(JobStatus.PUSHING_OUTPUT) new_file = gc.uploadFileToFolder(folderId, output_path) - gc.addMetadataToItem(new_file['itemId'], {"codec": "h264"}) + gc.addMetadataToItem( + new_file['itemId'], + { + "source_video": False, + "transcoder": "ffmpeg", + "codec": "h264", + }, + ) gc.addMetadataToItem( itemId, { diff --git a/server/dive_tasks/utils.py b/server/dive_tasks/utils.py index 3aa27b5cc..0d649ebf2 100644 --- a/server/dive_tasks/utils.py +++ b/server/dive_tasks/utils.py @@ -70,24 +70,24 @@ def organize_folder_for_training( return groundtruth -def get_source_video_filename(folderId: str, girder_client: GirderClient): +def get_video_filename(folderId: str, girder_client: GirderClient): """ - Searches a folderId for source videos that are compatible with training/pipelines - Will look for {"source_video":True} metadata first, then fall back to the converted video - indicated by {"codec":"h264"} - If neither found it will return None + Searches a folderId for videos that are compatible with training/pipelines - :folderId: Current path to wehere the items sit + * look for {"codec": 'h264', "source_video": False | None }, a transcoded video + * then fall back to {"source_video": True}, the user uploaded video + * If neither found it will return None + :folderId: Current path to where the items sit :girder_client: girder_client used to request the data """ - folder_contents = girder_client.listItem(folderId) backup_converted_file = None for item in folder_contents: file_name = item.get("name") - if item.get("meta", {}).get("source_video") is True: - return file_name - if item.get("meta", {}).get("codec") == "h264": + meta = item.get("meta", {}) + if meta.get("source_video") is True: backup_converted_file = file_name + elif meta.get("codec") == "h264": + return file_name return backup_converted_file