From cdf175f16c854a48f3b44da4f93cc431ba2d373b Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 4 Jan 2019 11:36:14 -0500 Subject: [PATCH] Added skippostprocess parameter, converted tabs to spaces in files --- docs/index.adoc | 4 +- docs/swagger.json | 2 +- index.js | 10 +- libs/Directories.js | 6 +- libs/Task.js | 978 +++++++++++++++++++++--------------------- libs/TaskManager.js | 464 ++++++++++---------- libs/logger.js | 26 +- libs/odmInfo.js | 498 ++++++++++----------- libs/odmRunner.js | 216 +++++----- libs/processRunner.js | 80 ++-- libs/statusCodes.js | 10 +- libs/utils.js | 28 +- package.json | 2 +- public/index.html | 29 +- public/js/main.js | 5 + 15 files changed, 1193 insertions(+), 1165 deletions(-) diff --git a/docs/index.adoc b/docs/index.adoc index a02d0f93..5ca641b4 100644 --- a/docs/index.adoc +++ b/docs/index.adoc @@ -8,7 +8,7 @@ REST API to access ODM === Version information [%hardbreaks] -_Version_ : 1.3.0 +_Version_ : 1.3.1 === Contact information @@ -299,6 +299,8 @@ _optional_|Images to process, plus an optional GCP file. If included, the GCP fi _optional_|An optional name to be associated with the task|string| |*FormData*|*options* + _optional_|Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, …]. For example, [{"name":"cmvs-maxImages","value":"500"},{"name":"time","value":true}]. For a list of all options, call /options|string| +|*FormData*|*skipPostProcessing* + +_optional_|When set, skips generation of map tiles, derivate assets, point cloud tiles.|boolean| |*FormData*|*zipurl* + _optional_|URL of the zip file containing the images to process, plus an optional GCP file. If included, the GCP file should have .txt extension|string| |=== diff --git a/docs/swagger.json b/docs/swagger.json index 079d14e3..65f8505e 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -1 +1 @@ -{"info":{"title":"node-opendronemap","version":"1.3.0","description":"REST API to access ODM","license":{"name":"GPL-3.0"},"contact":{"name":"Piero Toffanin"}},"consumes":["application/json"],"produces":["application/json","application/zip"],"basePath":"/","schemes":["http"],"swagger":"2.0","paths":{"/task/new":{"post":{"description":"Creates a new task and places it at the end of the processing queue","tags":["task"],"consumes":["multipart/form-data"],"parameters":[{"name":"images","in":"formData","description":"Images to process, plus an optional GCP file. If included, the GCP file should have .txt extension","required":false,"type":"file"},{"name":"zipurl","in":"formData","description":"URL of the zip file containing the images to process, plus an optional GCP file. If included, the GCP file should have .txt extension","required":false,"type":"string"},{"name":"name","in":"formData","description":"An optional name to be associated with the task","required":false,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"},{"name":"set-uuid","in":"header","description":"An optional UUID string that will be used as UUID for this task instead of generating a random one.","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/info":{"get":{"description":"Gets information about this task, such as name, creation date, processing time, status, command line options and number of images being processed. See schema definition for a full list.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Task Information","schema":{"title":"TaskInfo","type":"object","required":["uuid","name","dateCreated","processingTime","status","options","imagesCount"],"properties":{"uuid":{"type":"string","description":"UUID"},"name":{"type":"string","description":"Name"},"dateCreated":{"type":"integer","description":"Timestamp"},"processingTime":{"type":"integer","description":"Milliseconds that have elapsed since the task started being processed."},"status":{"type":"integer","description":"Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)","enum":[10,20,30,40,50]},"options":{"type":"array","description":"List of options used to process this task","items":{"type":"object","required":["name","value"],"properties":{"name":{"type":"string","description":"Option name (example: \"odm_meshing-octreeDepth\")"},"value":{"type":"string","description":"Value (example: 9)"}}}},"imagesCount":{"type":"integer","description":"Number of images"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/output":{"get":{"description":"Retrieves the console output of the OpenDroneMap's process. Useful for monitoring execution and to provide updates to the user.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"line","in":"query","description":"Optional line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. Defaults to 0 (retrieve all console output).","default":0,"required":false,"type":"integer"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Console Output","schema":{"type":"string"}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/download/{asset}":{"get":{"description":"Retrieves an asset (the output of OpenDroneMap's processing) associated with a task","tags":["task"],"produces":["application/zip"],"parameters":[{"name":"uuid","in":"path","type":"string","description":"UUID of the task","required":true},{"name":"asset","in":"path","type":"string","description":"Type of asset to download. Use \"all.zip\" for zip file containing all assets.","required":true,"enum":["all.zip","orthophoto.tif"]},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Asset File","schema":{"type":"file"}},"default":{"description":"Error message","schema":{"$ref":"#/definitions/Error"}}}}},"/task/cancel":{"post":{"description":"Cancels a task (stops its execution, or prevents it from being executed)","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/remove":{"post":{"description":"Removes a task and deletes all of its assets","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/restart":{"post":{"description":"Restarts a task that was previously canceled, that had failed to process or that successfully completed","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"options","in":"body","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options. Overrides the previous options set for this task.","required":false,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/options":{"get":{"description":"Retrieves the command line options that can be passed to process a task","parameters":[{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"tags":["server"],"responses":{"200":{"description":"Options","schema":{"type":"array","items":{"title":"Option","type":"object","required":["name","type","value","domain","help"],"properties":{"name":{"type":"string","description":"Command line option (exactly as it is passed to the OpenDroneMap process, minus the leading '--')"},"type":{"type":"string","description":"Datatype of the value of this option","enum":["int","float","string","bool"]},"value":{"type":"string","description":"Default value of this option"},"domain":{"type":"string","description":"Valid range of values (for example, \"positive integer\" or \"float > 0.0\")"},"help":{"type":"string","description":"Description of what this option does"}}}}}}}},"/info":{"get":{"description":"Retrieves information about this node","parameters":[{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"tags":["server"],"responses":{"200":{"description":"Info","schema":{"type":"object","required":["version","taskQueueCount"],"properties":{"version":{"type":"string","description":"Current API version"},"taskQueueCount":{"type":"integer","description":"Number of tasks currently being processed or waiting to be processed"},"availableMemory":{"type":"integer","description":"Amount of RAM available in bytes"},"totalMemory":{"type":"integer","description":"Amount of total RAM in the system in bytes"},"cpuCores":{"type":"integer","description":"Number of CPU cores (virtual)"},"maxImages":{"type":"integer","description":"Maximum number of images allowed for new tasks or null if there's no limit."},"maxParallelTasks":{"type":"integer","description":"Maximum number of tasks that can be processed simultaneously"},"odmVersion":{"type":"string","description":"Current version of ODM"}}}}}}},"/auth/info":{"get":{"description":"Retrieves login information for this node.","tags":["auth"],"responses":{"200":{"description":"LoginInformation","schema":{"type":"object","required":["message","loginUrl","registerUrl"],"properties":{"instructions":{"type":"string","description":"Message to be displayed to the user prior to login/registration. This might include instructions on how to register or login, or to communicate that authentication is not available."},"loginUrl":{"type":"string","description":"URL (absolute or relative) where to make a POST request to obtain a token, or null if login is disabled."},"registerUrl":{"type":"string","description":"URL (absolute or relative) where to make a POST request to register a user, or null if registration is disabled."}}}}}}},"/auth/login":{"post":{"description":"Retrieve a token from a username/password pair.","parameters":[{"name":"username","in":"body","description":"Username","required":true,"schema":{"type":"string"}},{"name":"password","in":"body","description":"Password","required":true,"type":"string"}],"responses":{"200":{"description":"Login Succeeded","schema":{"type":"object","required":["token"],"properties":{"token":{"type":"string","description":"Token to be passed as a query parameter to other API calls."}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/auth/register":{"post":{"description":"Register a new username/password.","parameters":[{"name":"username","in":"body","description":"Username","required":true,"schema":{"type":"string"}},{"name":"password","in":"body","description":"Password","required":true,"type":"string"}],"responses":{"200":{"description":"Response","schema":{"$ref":"#/definitions/Response"}}}}}},"definitions":{"Error":{"type":"object","required":["error"],"properties":{"error":{"type":"string","description":"Description of the error"}}},"Response":{"type":"object","required":["success"],"properties":{"success":{"type":"boolean","description":"true if the command succeeded, false otherwise"},"error":{"type":"string","description":"Error message if an error occured"}}}},"responses":{},"parameters":{},"securityDefinitions":{},"tags":[]} \ No newline at end of file +{"info":{"title":"node-opendronemap","version":"1.3.1","description":"REST API to access ODM","license":{"name":"GPL-3.0"},"contact":{"name":"Piero Toffanin"}},"consumes":["application/json"],"produces":["application/json","application/zip"],"basePath":"/","schemes":["http"],"swagger":"2.0","paths":{"/task/new":{"post":{"description":"Creates a new task and places it at the end of the processing queue","tags":["task"],"consumes":["multipart/form-data"],"parameters":[{"name":"images","in":"formData","description":"Images to process, plus an optional GCP file. If included, the GCP file should have .txt extension","required":false,"type":"file"},{"name":"zipurl","in":"formData","description":"URL of the zip file containing the images to process, plus an optional GCP file. If included, the GCP file should have .txt extension","required":false,"type":"string"},{"name":"name","in":"formData","description":"An optional name to be associated with the task","required":false,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"},{"name":"skipPostProcessing","in":"formData","description":"When set, skips generation of map tiles, derivate assets, point cloud tiles.","required":false,"type":"boolean"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"},{"name":"set-uuid","in":"header","description":"An optional UUID string that will be used as UUID for this task instead of generating a random one.","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/info":{"get":{"description":"Gets information about this task, such as name, creation date, processing time, status, command line options and number of images being processed. See schema definition for a full list.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Task Information","schema":{"title":"TaskInfo","type":"object","required":["uuid","name","dateCreated","processingTime","status","options","imagesCount"],"properties":{"uuid":{"type":"string","description":"UUID"},"name":{"type":"string","description":"Name"},"dateCreated":{"type":"integer","description":"Timestamp"},"processingTime":{"type":"integer","description":"Milliseconds that have elapsed since the task started being processed."},"status":{"type":"integer","description":"Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)","enum":[10,20,30,40,50]},"options":{"type":"array","description":"List of options used to process this task","items":{"type":"object","required":["name","value"],"properties":{"name":{"type":"string","description":"Option name (example: \"odm_meshing-octreeDepth\")"},"value":{"type":"string","description":"Value (example: 9)"}}}},"imagesCount":{"type":"integer","description":"Number of images"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/output":{"get":{"description":"Retrieves the console output of the OpenDroneMap's process. Useful for monitoring execution and to provide updates to the user.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"line","in":"query","description":"Optional line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. Defaults to 0 (retrieve all console output).","default":0,"required":false,"type":"integer"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Console Output","schema":{"type":"string"}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/download/{asset}":{"get":{"description":"Retrieves an asset (the output of OpenDroneMap's processing) associated with a task","tags":["task"],"produces":["application/zip"],"parameters":[{"name":"uuid","in":"path","type":"string","description":"UUID of the task","required":true},{"name":"asset","in":"path","type":"string","description":"Type of asset to download. Use \"all.zip\" for zip file containing all assets.","required":true,"enum":["all.zip","orthophoto.tif"]},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Asset File","schema":{"type":"file"}},"default":{"description":"Error message","schema":{"$ref":"#/definitions/Error"}}}}},"/task/cancel":{"post":{"description":"Cancels a task (stops its execution, or prevents it from being executed)","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/remove":{"post":{"description":"Removes a task and deletes all of its assets","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/restart":{"post":{"description":"Restarts a task that was previously canceled, that had failed to process or that successfully completed","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"options","in":"body","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options. Overrides the previous options set for this task.","required":false,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/options":{"get":{"description":"Retrieves the command line options that can be passed to process a task","parameters":[{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"tags":["server"],"responses":{"200":{"description":"Options","schema":{"type":"array","items":{"title":"Option","type":"object","required":["name","type","value","domain","help"],"properties":{"name":{"type":"string","description":"Command line option (exactly as it is passed to the OpenDroneMap process, minus the leading '--')"},"type":{"type":"string","description":"Datatype of the value of this option","enum":["int","float","string","bool"]},"value":{"type":"string","description":"Default value of this option"},"domain":{"type":"string","description":"Valid range of values (for example, \"positive integer\" or \"float > 0.0\")"},"help":{"type":"string","description":"Description of what this option does"}}}}}}}},"/info":{"get":{"description":"Retrieves information about this node","parameters":[{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"tags":["server"],"responses":{"200":{"description":"Info","schema":{"type":"object","required":["version","taskQueueCount"],"properties":{"version":{"type":"string","description":"Current API version"},"taskQueueCount":{"type":"integer","description":"Number of tasks currently being processed or waiting to be processed"},"availableMemory":{"type":"integer","description":"Amount of RAM available in bytes"},"totalMemory":{"type":"integer","description":"Amount of total RAM in the system in bytes"},"cpuCores":{"type":"integer","description":"Number of CPU cores (virtual)"},"maxImages":{"type":"integer","description":"Maximum number of images allowed for new tasks or null if there's no limit."},"maxParallelTasks":{"type":"integer","description":"Maximum number of tasks that can be processed simultaneously"},"odmVersion":{"type":"string","description":"Current version of ODM"}}}}}}},"/auth/info":{"get":{"description":"Retrieves login information for this node.","tags":["auth"],"responses":{"200":{"description":"LoginInformation","schema":{"type":"object","required":["message","loginUrl","registerUrl"],"properties":{"instructions":{"type":"string","description":"Message to be displayed to the user prior to login/registration. This might include instructions on how to register or login, or to communicate that authentication is not available."},"loginUrl":{"type":"string","description":"URL (absolute or relative) where to make a POST request to obtain a token, or null if login is disabled."},"registerUrl":{"type":"string","description":"URL (absolute or relative) where to make a POST request to register a user, or null if registration is disabled."}}}}}}},"/auth/login":{"post":{"description":"Retrieve a token from a username/password pair.","parameters":[{"name":"username","in":"body","description":"Username","required":true,"schema":{"type":"string"}},{"name":"password","in":"body","description":"Password","required":true,"type":"string"}],"responses":{"200":{"description":"Login Succeeded","schema":{"type":"object","required":["token"],"properties":{"token":{"type":"string","description":"Token to be passed as a query parameter to other API calls."}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/auth/register":{"post":{"description":"Register a new username/password.","parameters":[{"name":"username","in":"body","description":"Username","required":true,"schema":{"type":"string"}},{"name":"password","in":"body","description":"Password","required":true,"type":"string"}],"responses":{"200":{"description":"Response","schema":{"$ref":"#/definitions/Response"}}}}}},"definitions":{"Error":{"type":"object","required":["error"],"properties":{"error":{"type":"string","description":"Description of the error"}}},"Response":{"type":"object","required":["success"],"properties":{"success":{"type":"boolean","description":"true if the command succeeded, false otherwise"},"error":{"type":"string","description":"Error message if an error occured"}}}},"responses":{},"parameters":{},"securityDefinitions":{},"tags":[]} \ No newline at end of file diff --git a/index.js b/index.js index 18cfbd46..f756752b 100644 --- a/index.js +++ b/index.js @@ -119,6 +119,12 @@ let server; * required: false * type: string * - + * name: skipPostProcessing + * in: formData + * description: 'When set, skips generation of map tiles, derivate assets, point cloud tiles.' + * required: false + * type: boolean + * - * name: token * in: query * description: 'Token required for authentication (when authentication is required).' @@ -287,7 +293,9 @@ app.post('/task/new', authCheck, (req, res, next) => { res.json({ uuid: req.id }); cb(); } - }, req.body.options, req.body.webhook); + }, req.body.options, + req.body.webhook, + req.body.skipPostProcessing === 'true'); } ], err => { if (err) die(err.message); diff --git a/libs/Directories.js b/libs/Directories.js index 92e2af75..c63e3877 100644 --- a/libs/Directories.js +++ b/libs/Directories.js @@ -20,9 +20,9 @@ let config = require('../config'); let path = require('path'); class Directories{ - static get data(){ - return !config.test ? "data" : path.join("tests", "data"); - } + static get data(){ + return !config.test ? "data" : path.join("tests", "data"); + } } module.exports = Directories; \ No newline at end of file diff --git a/libs/Task.js b/libs/Task.js index e3e707cb..fcc59198 100644 --- a/libs/Task.js +++ b/libs/Task.js @@ -36,492 +36,494 @@ const request = require('request'); const statusCodes = require('./statusCodes'); module.exports = class Task{ - constructor(uuid, name, done, options = [], webhook = null){ - assert(uuid !== undefined, "uuid must be set"); - assert(done !== undefined, "ready must be set"); - - this.uuid = uuid; - this.name = name !== "" ? name : "Task of " + (new Date()).toISOString(); - this.dateCreated = new Date().getTime(); - this.processingTime = -1; - this.setStatus(statusCodes.QUEUED); - this.options = options; - this.gpcFiles = []; - this.output = []; - this.runningProcesses = []; - this.webhook = webhook; - - async.series([ - // Read images info - cb => { - fs.readdir(this.getImagesFolderPath(), (err, files) => { - if (err) cb(err); - else{ - this.images = files; - logger.debug(`Found ${this.images.length} images for ${this.uuid}`); - cb(null); - } - }); - }, - - // Find GCP (if any) - cb => { - fs.readdir(this.getGpcFolderPath(), (err, files) => { - if (err) cb(err); - else{ - files.forEach(file => { - if (/\.txt$/gi.test(file)){ - this.gpcFiles.push(file); - } - }); - logger.debug(`Found ${this.gpcFiles.length} GPC files (${this.gpcFiles.join(" ")}) for ${this.uuid}`); - cb(null); - } - }); - } - ], err => { - done(err, this); - }); - } - - static CreateFromSerialized(taskJson, done){ - new Task(taskJson.uuid, taskJson.name, (err, task) => { - if (err) done(err); - else{ - // Override default values with those - // provided in the taskJson - for (let k in taskJson){ - task[k] = taskJson[k]; - } - - // Tasks that were running should be put back to QUEUED state - if (task.status.code === statusCodes.RUNNING){ - task.status.code = statusCodes.QUEUED; - } - done(null, task); - } - }, taskJson.options, taskJson.webhook); - } - - // Get path where images are stored for this task - // (relative to nodejs process CWD) - getImagesFolderPath(){ - return path.join(this.getProjectFolderPath(), "images"); - } - - // Get path where GPC file(s) are stored - // (relative to nodejs process CWD) - getGpcFolderPath(){ - return path.join(this.getProjectFolderPath(), "gpc"); - } - - // Get path of project (where all images and assets folder are contained) - // (relative to nodejs process CWD) - getProjectFolderPath(){ - return path.join(Directories.data, this.uuid); - } - - // Get the path of the archive where all assets - // outputted by this task are stored. - getAssetsArchivePath(filename){ - if (filename == 'all.zip'){ - // OK, do nothing - }else if (filename == 'orthophoto.tif'){ - if (config.test){ - if (config.testSkipOrthophotos) return false; - else filename = path.join('..', '..', 'processing_results', 'odm_orthophoto', `odm_${filename}`); - }else{ - filename = path.join('odm_orthophoto', `odm_${filename}`); - } - }else{ - return false; // Invalid - } - - return path.join(this.getProjectFolderPath(), filename); - } - - // Deletes files and folders related to this task - cleanup(cb){ - rmdir(this.getProjectFolderPath(), cb); - } - - setStatus(code, extra){ - this.status = { - code: code - }; - for (let k in extra){ - this.status[k] = extra[k]; - } - } - - updateProcessingTime(resetTime){ - this.processingTime = resetTime ? - -1 : - new Date().getTime() - this.dateCreated; - } - - startTrackingProcessingTime(){ - this.updateProcessingTime(); - if (!this._updateProcessingTimeInterval){ - this._updateProcessingTimeInterval = setInterval(() => { - this.updateProcessingTime(); - }, 1000); - } - } - - stopTrackingProcessingTime(resetTime){ - this.updateProcessingTime(resetTime); - if (this._updateProcessingTimeInterval){ - clearInterval(this._updateProcessingTimeInterval); - this._updateProcessingTimeInterval = null; - } - } - - getStatus(){ - return this.status.code; - } - - isCanceled(){ - return this.status.code === statusCodes.CANCELED; - } - - // Cancels the current task (unless it's already canceled) - cancel(cb){ - if (this.status.code !== statusCodes.CANCELED){ - let wasRunning = this.status.code === statusCodes.RUNNING; - this.setStatus(statusCodes.CANCELED); - - if (wasRunning){ - this.runningProcesses.forEach(proc => { - // TODO: this does NOT guarantee that - // the process will immediately terminate. - // For eaxmple in the case of the ODM process, the process will continue running for a while - // This might need to be fixed on ODM's end. - - // During testing, proc is undefined - if (proc) kill(proc.pid); - }); - this.runningProcesses = []; - } - - this.stopTrackingProcessingTime(true); - cb(null); - }else{ - cb(new Error("Task already cancelled")); - } - } - - // Starts processing the task with OpenDroneMap - // This will spawn a new process. - start(done){ - const finished = err => { - this.stopTrackingProcessingTime(); - done(err); - }; - - const postProcess = () => { - const createZipArchive = (outputFilename, files) => { - return (done) => { - this.output.push(`Compressing ${outputFilename}\n`); - - let output = fs.createWriteStream(this.getAssetsArchivePath(outputFilename)); - let archive = archiver.create('zip', { - zlib: { level: 1 } // Sets the compression level (1 = best speed since most assets are already compressed) - }); - - archive.on('finish', () => { - // TODO: is this being fired twice? - done(); - }); - - archive.on('error', err => { - logger.error(`Could not archive .zip file: ${err.message}`); - done(err); - }); - - archive.pipe(output); - let globs = []; - - const sourcePath = !config.test ? - this.getProjectFolderPath() : - path.join("tests", "processing_results"); - - // Process files and directories first - files.forEach(file => { - let filePath = path.join(sourcePath, file); - - // Skip non-existing items - if (!fs.existsSync(filePath)) return; - - let isGlob = /\*/.test(file), - isDirectory = !isGlob && fs.lstatSync(filePath).isDirectory(); - - if (isDirectory){ - archive.directory(filePath, file); - }else if (isGlob){ - globs.push(filePath); - }else{ - archive.file(filePath, {name: file}); - } - }); - - // Check for globs - if (globs.length !== 0){ - let pending = globs.length; - - globs.forEach(pattern => { - glob(pattern, (err, files) => { - if (err) done(err); - else{ - files.forEach(file => { - if (fs.lstatSync(file).isFile()){ - archive.file(file, {name: path.basename(file)}); - }else{ - logger.debug(`Could not add ${file} from glob`); - } - }); - - if (--pending === 0){ - archive.finalize(); - } - } - }); - }); - }else{ - archive.finalize(); - } - }; - }; - - const runPostProcessingScript = () => { - return (done) => { - this.runningProcesses.push( - processRunner.runPostProcessingScript({ - projectFolderPath: this.getProjectFolderPath() - }, (err, code, signal) => { - if (err) done(err); - else{ - if (code === 0) done(); - else done(new Error(`Process exited with code ${code}`)); - } - }, output => { - this.output.push(output); - }) - ); - }; - }; - - // All paths are relative to the project directory (./data//) - let allPaths = ['odm_orthophoto', 'odm_georeferencing', 'odm_texturing', - 'odm_dem/dsm.tif', 'odm_dem/dtm.tif', 'dsm_tiles', 'dtm_tiles', - 'odm_meshing', 'orthophoto_tiles', 'potree_pointcloud', 'images.json']; - - if (config.test){ - if (config.testSkipOrthophotos){ - logger.info("Test mode will skip orthophoto generation"); - - // Exclude these folders from the all.zip archive - ['odm_orthophoto', 'orthophoto_tiles'].forEach(dir => { - allPaths.splice(allPaths.indexOf(dir), 1); - }); - } - - if (config.testSkipDems){ - logger.info("Test mode will skip DEMs generation"); - - // Exclude these folders from the all.zip archive - ['odm_dem/dsm.tif', 'odm_dem/dtm.tif', 'dsm_tiles', 'dtm_tiles'].forEach(p => { - allPaths.splice(allPaths.indexOf(p), 1); - }); - } - } - - let tasks = [ - runPostProcessingScript(), - createZipArchive('all.zip', allPaths) - ]; - - // Upload to S3 all paths + all.zip file (if config says so) - if (S3.enabled()){ - tasks.push((done) => { - let s3Paths; - if (config.test){ - s3Paths = ['all.zip']; // During testing only upload all.zip - }else if (config.s3UploadEverything){ - s3Paths = ['all.zip'].concat(allPaths); - }else{ - s3Paths = ['all.zip', 'odm_orthophoto/odm_orthophoto.tif']; - } - - S3.uploadPaths(this.getProjectFolderPath(), config.s3Bucket, this.uuid, s3Paths, - err => { - if (!err) this.output.push("Done uploading to S3!"); - done(err); - }, output => this.output.push(output)); - }); - } - - async.series(tasks, (err) => { - if (!err){ - this.setStatus(statusCodes.COMPLETED); - finished(); - }else{ - this.setStatus(statusCodes.FAILED); - finished(err); - } - }); - }; - - if (this.status.code === statusCodes.QUEUED){ - this.startTrackingProcessingTime(); - this.setStatus(statusCodes.RUNNING); - - let runnerOptions = this.options.reduce((result, opt) => { - result[opt.name] = opt.value; - return result; - }, {}); - - runnerOptions["project-path"] = fs.realpathSync(Directories.data); - - if (this.gpcFiles.length > 0){ - runnerOptions.gcp = fs.realpathSync(path.join(this.getGpcFolderPath(), this.gpcFiles[0])); - } - - this.runningProcesses.push(odmRunner.run(runnerOptions, this.uuid, (err, code, signal) => { - if (err){ - this.setStatus(statusCodes.FAILED, {errorMessage: `Could not start process (${err.message})`}); - finished(err); - }else{ - // Don't evaluate if we caused the process to exit via SIGINT? - if (this.status.code !== statusCodes.CANCELED){ - if (code === 0){ - postProcess(); - }else{ - this.setStatus(statusCodes.FAILED, {errorMessage: `Process exited with code ${code}`}); - finished(); - } - }else{ - finished(); - } - } - }, output => { - // Replace console colors - output = output.replace(/\x1b\[[0-9;]*m/g, ""); - - // Split lines and trim - output.trim().split('\n').forEach(line => { - this.output.push(line.trim()); - }); - }) - ); - - return true; - }else{ - return false; - } - } - - // Re-executes the task (by setting it's state back to QUEUED) - // Only tasks that have been canceled, completed or have failed can be restarted. - restart(options, cb){ - if ([statusCodes.CANCELED, statusCodes.FAILED, statusCodes.COMPLETED].indexOf(this.status.code) !== -1){ - this.setStatus(statusCodes.QUEUED); - this.dateCreated = new Date().getTime(); - this.output = []; - this.stopTrackingProcessingTime(true); - if (options !== undefined) this.options = options; - cb(null); - }else{ - cb(new Error("Task cannot be restarted")); - } - } - - // Returns the description of the task. - getInfo(){ - return { - uuid: this.uuid, - name: this.name, - dateCreated: this.dateCreated, - processingTime: this.processingTime, - status: this.status, - options: this.options, - imagesCount: this.images.length - }; - } - - // Returns the output of the OpenDroneMap process - // Optionally starting from a certain line number - getOutput(startFromLine = 0){ - return this.output.slice(startFromLine, this.output.length); - } - - // Reads the contents of the tasks's - // images.json and returns its JSON representation - readImagesDatabase(callback){ - const imagesDbPath = !config.test ? - path.join(this.getProjectFolderPath(), 'images.json') : - path.join('tests', 'processing_results', 'images.json'); - - fs.readFile(imagesDbPath, 'utf8', (err, data) => { - if (err) callback(err); - else{ - try{ - const json = JSON.parse(data); - callback(null, json); - }catch(e){ - callback(e); - } - } - }); - } - - callWebhooks(){ - // Hooks can be passed via command line - // or for each individual task - const hooks = [this.webhook, config.webhook]; - - this.readImagesDatabase((err, images) => { - if (err) logger.warn(err); // Continue with callback - if (!images) images = []; - - let json = this.getInfo(); - json.images = images; - - hooks.forEach(hook => { - if (hook && hook.length > 3){ - const notifyCallback = (attempt) => { - if (attempt > 5){ - logger.warn(`Webhook invokation failed, will not retry: ${hook}`); - return; - } - request.post(hook, { json }, - (error, response) => { - if (error || response.statusCode != 200){ - logger.warn(`Webhook invokation failed, will retry in a bit: ${hook}`); - setTimeout(() => { - notifyCallback(attempt + 1); - }, attempt * 5000); - }else{ - logger.debug(`Webhook invoked: ${hook}`); - } - }); - }; - notifyCallback(0); - } - }); - }); - } - - // Returns the data necessary to serialize this - // task to restore it later. - serialize(){ - return { - uuid: this.uuid, - name: this.name, - dateCreated: this.dateCreated, - status: this.status, - options: this.options, - webhook: this.webhook - }; - } + constructor(uuid, name, done, options = [], webhook = null, skipPostProcessing = false){ + assert(uuid !== undefined, "uuid must be set"); + assert(done !== undefined, "ready must be set"); + + this.uuid = uuid; + this.name = name !== "" ? name : "Task of " + (new Date()).toISOString(); + this.dateCreated = new Date().getTime(); + this.processingTime = -1; + this.setStatus(statusCodes.QUEUED); + this.options = options; + this.gpcFiles = []; + this.output = []; + this.runningProcesses = []; + this.webhook = webhook; + this.skipPostProcessing = skipPostProcessing; + + async.series([ + // Read images info + cb => { + fs.readdir(this.getImagesFolderPath(), (err, files) => { + if (err) cb(err); + else{ + this.images = files; + logger.debug(`Found ${this.images.length} images for ${this.uuid}`); + cb(null); + } + }); + }, + + // Find GCP (if any) + cb => { + fs.readdir(this.getGpcFolderPath(), (err, files) => { + if (err) cb(err); + else{ + files.forEach(file => { + if (/\.txt$/gi.test(file)){ + this.gpcFiles.push(file); + } + }); + logger.debug(`Found ${this.gpcFiles.length} GPC files (${this.gpcFiles.join(" ")}) for ${this.uuid}`); + cb(null); + } + }); + } + ], err => { + done(err, this); + }); + } + + static CreateFromSerialized(taskJson, done){ + new Task(taskJson.uuid, taskJson.name, (err, task) => { + if (err) done(err); + else{ + // Override default values with those + // provided in the taskJson + for (let k in taskJson){ + task[k] = taskJson[k]; + } + + // Tasks that were running should be put back to QUEUED state + if (task.status.code === statusCodes.RUNNING){ + task.status.code = statusCodes.QUEUED; + } + done(null, task); + } + }, taskJson.options, taskJson.webhook, taskJson.skipPostProcessing); + } + + // Get path where images are stored for this task + // (relative to nodejs process CWD) + getImagesFolderPath(){ + return path.join(this.getProjectFolderPath(), "images"); + } + + // Get path where GPC file(s) are stored + // (relative to nodejs process CWD) + getGpcFolderPath(){ + return path.join(this.getProjectFolderPath(), "gpc"); + } + + // Get path of project (where all images and assets folder are contained) + // (relative to nodejs process CWD) + getProjectFolderPath(){ + return path.join(Directories.data, this.uuid); + } + + // Get the path of the archive where all assets + // outputted by this task are stored. + getAssetsArchivePath(filename){ + if (filename == 'all.zip'){ + // OK, do nothing + }else if (filename == 'orthophoto.tif'){ + if (config.test){ + if (config.testSkipOrthophotos) return false; + else filename = path.join('..', '..', 'processing_results', 'odm_orthophoto', `odm_${filename}`); + }else{ + filename = path.join('odm_orthophoto', `odm_${filename}`); + } + }else{ + return false; // Invalid + } + + return path.join(this.getProjectFolderPath(), filename); + } + + // Deletes files and folders related to this task + cleanup(cb){ + rmdir(this.getProjectFolderPath(), cb); + } + + setStatus(code, extra){ + this.status = { + code: code + }; + for (let k in extra){ + this.status[k] = extra[k]; + } + } + + updateProcessingTime(resetTime){ + this.processingTime = resetTime ? + -1 : + new Date().getTime() - this.dateCreated; + } + + startTrackingProcessingTime(){ + this.updateProcessingTime(); + if (!this._updateProcessingTimeInterval){ + this._updateProcessingTimeInterval = setInterval(() => { + this.updateProcessingTime(); + }, 1000); + } + } + + stopTrackingProcessingTime(resetTime){ + this.updateProcessingTime(resetTime); + if (this._updateProcessingTimeInterval){ + clearInterval(this._updateProcessingTimeInterval); + this._updateProcessingTimeInterval = null; + } + } + + getStatus(){ + return this.status.code; + } + + isCanceled(){ + return this.status.code === statusCodes.CANCELED; + } + + // Cancels the current task (unless it's already canceled) + cancel(cb){ + if (this.status.code !== statusCodes.CANCELED){ + let wasRunning = this.status.code === statusCodes.RUNNING; + this.setStatus(statusCodes.CANCELED); + + if (wasRunning){ + this.runningProcesses.forEach(proc => { + // TODO: this does NOT guarantee that + // the process will immediately terminate. + // For eaxmple in the case of the ODM process, the process will continue running for a while + // This might need to be fixed on ODM's end. + + // During testing, proc is undefined + if (proc) kill(proc.pid); + }); + this.runningProcesses = []; + } + + this.stopTrackingProcessingTime(true); + cb(null); + }else{ + cb(new Error("Task already cancelled")); + } + } + + // Starts processing the task with OpenDroneMap + // This will spawn a new process. + start(done){ + const finished = err => { + this.stopTrackingProcessingTime(); + done(err); + }; + + const postProcess = () => { + const createZipArchive = (outputFilename, files) => { + return (done) => { + this.output.push(`Compressing ${outputFilename}\n`); + + let output = fs.createWriteStream(this.getAssetsArchivePath(outputFilename)); + let archive = archiver.create('zip', { + zlib: { level: 1 } // Sets the compression level (1 = best speed since most assets are already compressed) + }); + + archive.on('finish', () => { + // TODO: is this being fired twice? + done(); + }); + + archive.on('error', err => { + logger.error(`Could not archive .zip file: ${err.message}`); + done(err); + }); + + archive.pipe(output); + let globs = []; + + const sourcePath = !config.test ? + this.getProjectFolderPath() : + path.join("tests", "processing_results"); + + // Process files and directories first + files.forEach(file => { + let filePath = path.join(sourcePath, file); + + // Skip non-existing items + if (!fs.existsSync(filePath)) return; + + let isGlob = /\*/.test(file), + isDirectory = !isGlob && fs.lstatSync(filePath).isDirectory(); + + if (isDirectory){ + archive.directory(filePath, file); + }else if (isGlob){ + globs.push(filePath); + }else{ + archive.file(filePath, {name: file}); + } + }); + + // Check for globs + if (globs.length !== 0){ + let pending = globs.length; + + globs.forEach(pattern => { + glob(pattern, (err, files) => { + if (err) done(err); + else{ + files.forEach(file => { + if (fs.lstatSync(file).isFile()){ + archive.file(file, {name: path.basename(file)}); + }else{ + logger.debug(`Could not add ${file} from glob`); + } + }); + + if (--pending === 0){ + archive.finalize(); + } + } + }); + }); + }else{ + archive.finalize(); + } + }; + }; + + const runPostProcessingScript = () => { + return (done) => { + this.runningProcesses.push( + processRunner.runPostProcessingScript({ + projectFolderPath: this.getProjectFolderPath() + }, (err, code, signal) => { + if (err) done(err); + else{ + if (code === 0) done(); + else done(new Error(`Process exited with code ${code}`)); + } + }, output => { + this.output.push(output); + }) + ); + }; + }; + + // All paths are relative to the project directory (./data//) + let allPaths = ['odm_orthophoto', 'odm_georeferencing', 'odm_texturing', + 'odm_dem/dsm.tif', 'odm_dem/dtm.tif', 'dsm_tiles', 'dtm_tiles', + 'odm_meshing', 'orthophoto_tiles', 'potree_pointcloud', 'images.json']; + + if (config.test){ + if (config.testSkipOrthophotos){ + logger.info("Test mode will skip orthophoto generation"); + + // Exclude these folders from the all.zip archive + ['odm_orthophoto', 'orthophoto_tiles'].forEach(dir => { + allPaths.splice(allPaths.indexOf(dir), 1); + }); + } + + if (config.testSkipDems){ + logger.info("Test mode will skip DEMs generation"); + + // Exclude these folders from the all.zip archive + ['odm_dem/dsm.tif', 'odm_dem/dtm.tif', 'dsm_tiles', 'dtm_tiles'].forEach(p => { + allPaths.splice(allPaths.indexOf(p), 1); + }); + } + } + + let tasks = []; + if (!this.skipPostProcessing) tasks.push(runPostProcessingScript()); + + tasks.push(createZipArchive('all.zip', allPaths)); + + // Upload to S3 all paths + all.zip file (if config says so) + if (S3.enabled()){ + tasks.push((done) => { + let s3Paths; + if (config.test){ + s3Paths = ['all.zip']; // During testing only upload all.zip + }else if (config.s3UploadEverything){ + s3Paths = ['all.zip'].concat(allPaths); + }else{ + s3Paths = ['all.zip', 'odm_orthophoto/odm_orthophoto.tif']; + } + + S3.uploadPaths(this.getProjectFolderPath(), config.s3Bucket, this.uuid, s3Paths, + err => { + if (!err) this.output.push("Done uploading to S3!"); + done(err); + }, output => this.output.push(output)); + }); + } + + async.series(tasks, (err) => { + if (!err){ + this.setStatus(statusCodes.COMPLETED); + finished(); + }else{ + this.setStatus(statusCodes.FAILED); + finished(err); + } + }); + }; + + if (this.status.code === statusCodes.QUEUED){ + this.startTrackingProcessingTime(); + this.setStatus(statusCodes.RUNNING); + + let runnerOptions = this.options.reduce((result, opt) => { + result[opt.name] = opt.value; + return result; + }, {}); + + runnerOptions["project-path"] = fs.realpathSync(Directories.data); + + if (this.gpcFiles.length > 0){ + runnerOptions.gcp = fs.realpathSync(path.join(this.getGpcFolderPath(), this.gpcFiles[0])); + } + + this.runningProcesses.push(odmRunner.run(runnerOptions, this.uuid, (err, code, signal) => { + if (err){ + this.setStatus(statusCodes.FAILED, {errorMessage: `Could not start process (${err.message})`}); + finished(err); + }else{ + // Don't evaluate if we caused the process to exit via SIGINT? + if (this.status.code !== statusCodes.CANCELED){ + if (code === 0){ + postProcess(); + }else{ + this.setStatus(statusCodes.FAILED, {errorMessage: `Process exited with code ${code}`}); + finished(); + } + }else{ + finished(); + } + } + }, output => { + // Replace console colors + output = output.replace(/\x1b\[[0-9;]*m/g, ""); + + // Split lines and trim + output.trim().split('\n').forEach(line => { + this.output.push(line.trim()); + }); + }) + ); + + return true; + }else{ + return false; + } + } + + // Re-executes the task (by setting it's state back to QUEUED) + // Only tasks that have been canceled, completed or have failed can be restarted. + restart(options, cb){ + if ([statusCodes.CANCELED, statusCodes.FAILED, statusCodes.COMPLETED].indexOf(this.status.code) !== -1){ + this.setStatus(statusCodes.QUEUED); + this.dateCreated = new Date().getTime(); + this.output = []; + this.stopTrackingProcessingTime(true); + if (options !== undefined) this.options = options; + cb(null); + }else{ + cb(new Error("Task cannot be restarted")); + } + } + + // Returns the description of the task. + getInfo(){ + return { + uuid: this.uuid, + name: this.name, + dateCreated: this.dateCreated, + processingTime: this.processingTime, + status: this.status, + options: this.options, + imagesCount: this.images.length + }; + } + + // Returns the output of the OpenDroneMap process + // Optionally starting from a certain line number + getOutput(startFromLine = 0){ + return this.output.slice(startFromLine, this.output.length); + } + + // Reads the contents of the tasks's + // images.json and returns its JSON representation + readImagesDatabase(callback){ + const imagesDbPath = !config.test ? + path.join(this.getProjectFolderPath(), 'images.json') : + path.join('tests', 'processing_results', 'images.json'); + + fs.readFile(imagesDbPath, 'utf8', (err, data) => { + if (err) callback(err); + else{ + try{ + const json = JSON.parse(data); + callback(null, json); + }catch(e){ + callback(e); + } + } + }); + } + + callWebhooks(){ + // Hooks can be passed via command line + // or for each individual task + const hooks = [this.webhook, config.webhook]; + + this.readImagesDatabase((err, images) => { + if (err) logger.warn(err); // Continue with callback + if (!images) images = []; + + let json = this.getInfo(); + json.images = images; + + hooks.forEach(hook => { + if (hook && hook.length > 3){ + const notifyCallback = (attempt) => { + if (attempt > 5){ + logger.warn(`Webhook invokation failed, will not retry: ${hook}`); + return; + } + request.post(hook, { json }, + (error, response) => { + if (error || response.statusCode != 200){ + logger.warn(`Webhook invokation failed, will retry in a bit: ${hook}`); + setTimeout(() => { + notifyCallback(attempt + 1); + }, attempt * 5000); + }else{ + logger.debug(`Webhook invoked: ${hook}`); + } + }); + }; + notifyCallback(0); + } + }); + }); + } + + // Returns the data necessary to serialize this + // task to restore it later. + serialize(){ + return { + uuid: this.uuid, + name: this.name, + dateCreated: this.dateCreated, + status: this.status, + options: this.options, + webhook: this.webhook, + skipPostProcessing: !!this.skipPostProcessing + }; + } }; diff --git a/libs/TaskManager.js b/libs/TaskManager.js index 2528415b..9b1ced59 100644 --- a/libs/TaskManager.js +++ b/libs/TaskManager.js @@ -32,236 +32,236 @@ const TASKS_DUMP_FILE = path.join(Directories.data, "tasks.json"); const CLEANUP_TASKS_IF_OLDER_THAN = 1000 * 60 * config.cleanupTasksAfter; // minutes module.exports = class TaskManager{ - constructor(done){ - this.tasks = {}; - this.runningQueue = []; - - async.series([ - cb => this.restoreTaskListFromDump(cb), - cb => this.removeOldTasks(cb), - cb => this.removeOrphanedDirectories(cb), - cb => { - this.processNextTask(); - cb(); - }, - cb => { - // Every hour - schedule.scheduleJob('0 * * * *', () => { - this.removeOldTasks(); - this.dumpTaskList(); - }); - - cb(); - } - ], done); - } - - // Removes old tasks that have either failed, are completed, or - // have been canceled. - removeOldTasks(done){ - let list = []; - let now = new Date().getTime(); - logger.debug("Checking for old tasks to be removed..."); - - for (let uuid in this.tasks){ - let task = this.tasks[uuid]; - - if ([statusCodes.FAILED, - statusCodes.COMPLETED, - statusCodes.CANCELED].indexOf(task.status.code) !== -1 && - now - task.dateCreated > CLEANUP_TASKS_IF_OLDER_THAN){ - list.push(task.uuid); - } - } - - async.eachSeries(list, (uuid, cb) => { - logger.info(`Cleaning up old task ${uuid}`); - this.remove(uuid, cb); - }, done); - } - - // Removes directories that don't have a corresponding - // task associated with it (maybe as a cause of an abrupt exit) - removeOrphanedDirectories(done){ - logger.info("Checking for orphaned directories to be removed..."); - - fs.readdir(Directories.data, (err, entries) => { - if (err) done(err); - else{ - async.eachSeries(entries, (entry, cb) => { - let dirPath = path.join(Directories.data, entry); - if (fs.statSync(dirPath).isDirectory() && - entry.match(/^[\w\d]+\-[\w\d]+\-[\w\d]+\-[\w\d]+\-[\w\d]+$/) && - !this.tasks[entry]){ - logger.info(`Found orphaned directory: ${entry}, removing...`); - rmdir(dirPath, cb); - }else cb(); - }, done); - } - }); - } - - // Load tasks that already exists (if any) - restoreTaskListFromDump(done){ - fs.readFile(TASKS_DUMP_FILE, (err, data) => { - if (!err){ - let tasks; - try{ - tasks = JSON.parse(data.toString()); - }catch(e){ - done(new Error(`Could not load task list. It looks like the ${TASKS_DUMP_FILE} is corrupted (${e.message}). Please manually delete the file and try again.`)); - return; - } - - async.each(tasks, (taskJson, done) => { - Task.CreateFromSerialized(taskJson, (err, task) => { - if (err) done(err); - else{ - this.tasks[task.uuid] = task; - done(); - } - }); - }, err => { - logger.info(`Initialized ${tasks.length} tasks`); - if (done !== undefined) done(); - }); - }else{ - logger.info("No tasks dump found"); - if (done !== undefined) done(); - } - }); - } - - // Finds the first QUEUED task. - findNextTaskToProcess(){ - for (let uuid in this.tasks){ - if (this.tasks[uuid].getStatus() === statusCodes.QUEUED){ - return this.tasks[uuid]; - } - } - } - - // Finds the next tasks, adds them to the running queue, - // and starts the tasks (up to the limit). - processNextTask(){ - if (this.runningQueue.length < config.parallelQueueProcessing){ - let task = this.findNextTaskToProcess(); - if (task){ - this.addToRunningQueue(task); - task.start(() => { - - task.callWebhooks(); - - this.removeFromRunningQueue(task); - this.processNextTask(); - }); - - if (this.runningQueue.length < config.parallelQueueProcessing) this.processNextTask(); - } - }else{ - // Do nothing - } - } - - addToRunningQueue(task){ - assert(task.constructor.name === "Task", "Must be a Task object"); - this.runningQueue.push(task); - } - - removeFromRunningQueue(task){ - assert(task.constructor.name === "Task", "Must be a Task object"); - this.runningQueue = this.runningQueue.filter(t => t !== task); - } - - addNew(task){ - assert(task.constructor.name === "Task", "Must be a Task object"); - this.tasks[task.uuid] = task; - - this.processNextTask(); - } - - // Stops the execution of a task - // (without removing it from the system). - cancel(uuid, cb){ - let task = this.find(uuid, cb); - if (task){ - if (!task.isCanceled()){ - task.cancel(err => { - this.removeFromRunningQueue(task); - this.processNextTask(); - cb(err); - }); - }else{ - cb(null); // Nothing to be done - } - } - } - - // Removes a task from the system. - // Before being removed, the task is canceled. - remove(uuid, cb){ - this.cancel(uuid, err => { - if (!err){ - let task = this.find(uuid, cb); - if (task){ - task.cleanup(err => { - if (!err){ - delete(this.tasks[uuid]); - this.processNextTask(); - cb(null); - }else cb(err); - }); - }else; // cb is called by find on error - }else cb(err); - }); - } - - // Restarts (puts back into QUEUED state) - // a task that is either in CANCELED or FAILED state. - // When options is set, the task's options are overriden - restart(uuid, options, cb){ - let task = this.find(uuid, cb); - if (task){ - task.restart(options, err => { - if (!err) this.processNextTask(); - cb(err); - }); - } - } - - // Finds a task by its UUID string. - find(uuid, cb){ - let task = this.tasks[uuid]; - if (!task && cb) cb(new Error(`${uuid} not found`)); - return task; - } - - // Serializes the list of tasks and saves it - // to disk - dumpTaskList(done){ - let output = []; - - for (let uuid in this.tasks){ - output.push(this.tasks[uuid].serialize()); - } - - fs.writeFile(TASKS_DUMP_FILE, JSON.stringify(output), err => { - if (err) logger.error(`Could not dump tasks: ${err.message}`); - else logger.debug("Dumped tasks list."); - if (done !== undefined) done(); - }); - } - - getQueueCount(){ - let count = 0; - for (let uuid in this.tasks){ - let task = this.tasks[uuid]; - - if ([statusCodes.QUEUED, - statusCodes.RUNNING].indexOf(task.status.code) !== -1){ - count++; - } - } - return count; - } + constructor(done){ + this.tasks = {}; + this.runningQueue = []; + + async.series([ + cb => this.restoreTaskListFromDump(cb), + cb => this.removeOldTasks(cb), + cb => this.removeOrphanedDirectories(cb), + cb => { + this.processNextTask(); + cb(); + }, + cb => { + // Every hour + schedule.scheduleJob('0 * * * *', () => { + this.removeOldTasks(); + this.dumpTaskList(); + }); + + cb(); + } + ], done); + } + + // Removes old tasks that have either failed, are completed, or + // have been canceled. + removeOldTasks(done){ + let list = []; + let now = new Date().getTime(); + logger.debug("Checking for old tasks to be removed..."); + + for (let uuid in this.tasks){ + let task = this.tasks[uuid]; + + if ([statusCodes.FAILED, + statusCodes.COMPLETED, + statusCodes.CANCELED].indexOf(task.status.code) !== -1 && + now - task.dateCreated > CLEANUP_TASKS_IF_OLDER_THAN){ + list.push(task.uuid); + } + } + + async.eachSeries(list, (uuid, cb) => { + logger.info(`Cleaning up old task ${uuid}`); + this.remove(uuid, cb); + }, done); + } + + // Removes directories that don't have a corresponding + // task associated with it (maybe as a cause of an abrupt exit) + removeOrphanedDirectories(done){ + logger.info("Checking for orphaned directories to be removed..."); + + fs.readdir(Directories.data, (err, entries) => { + if (err) done(err); + else{ + async.eachSeries(entries, (entry, cb) => { + let dirPath = path.join(Directories.data, entry); + if (fs.statSync(dirPath).isDirectory() && + entry.match(/^[\w\d]+\-[\w\d]+\-[\w\d]+\-[\w\d]+\-[\w\d]+$/) && + !this.tasks[entry]){ + logger.info(`Found orphaned directory: ${entry}, removing...`); + rmdir(dirPath, cb); + }else cb(); + }, done); + } + }); + } + + // Load tasks that already exists (if any) + restoreTaskListFromDump(done){ + fs.readFile(TASKS_DUMP_FILE, (err, data) => { + if (!err){ + let tasks; + try{ + tasks = JSON.parse(data.toString()); + }catch(e){ + done(new Error(`Could not load task list. It looks like the ${TASKS_DUMP_FILE} is corrupted (${e.message}). Please manually delete the file and try again.`)); + return; + } + + async.each(tasks, (taskJson, done) => { + Task.CreateFromSerialized(taskJson, (err, task) => { + if (err) done(err); + else{ + this.tasks[task.uuid] = task; + done(); + } + }); + }, err => { + logger.info(`Initialized ${tasks.length} tasks`); + if (done !== undefined) done(); + }); + }else{ + logger.info("No tasks dump found"); + if (done !== undefined) done(); + } + }); + } + + // Finds the first QUEUED task. + findNextTaskToProcess(){ + for (let uuid in this.tasks){ + if (this.tasks[uuid].getStatus() === statusCodes.QUEUED){ + return this.tasks[uuid]; + } + } + } + + // Finds the next tasks, adds them to the running queue, + // and starts the tasks (up to the limit). + processNextTask(){ + if (this.runningQueue.length < config.parallelQueueProcessing){ + let task = this.findNextTaskToProcess(); + if (task){ + this.addToRunningQueue(task); + task.start(() => { + + task.callWebhooks(); + + this.removeFromRunningQueue(task); + this.processNextTask(); + }); + + if (this.runningQueue.length < config.parallelQueueProcessing) this.processNextTask(); + } + }else{ + // Do nothing + } + } + + addToRunningQueue(task){ + assert(task.constructor.name === "Task", "Must be a Task object"); + this.runningQueue.push(task); + } + + removeFromRunningQueue(task){ + assert(task.constructor.name === "Task", "Must be a Task object"); + this.runningQueue = this.runningQueue.filter(t => t !== task); + } + + addNew(task){ + assert(task.constructor.name === "Task", "Must be a Task object"); + this.tasks[task.uuid] = task; + + this.processNextTask(); + } + + // Stops the execution of a task + // (without removing it from the system). + cancel(uuid, cb){ + let task = this.find(uuid, cb); + if (task){ + if (!task.isCanceled()){ + task.cancel(err => { + this.removeFromRunningQueue(task); + this.processNextTask(); + cb(err); + }); + }else{ + cb(null); // Nothing to be done + } + } + } + + // Removes a task from the system. + // Before being removed, the task is canceled. + remove(uuid, cb){ + this.cancel(uuid, err => { + if (!err){ + let task = this.find(uuid, cb); + if (task){ + task.cleanup(err => { + if (!err){ + delete(this.tasks[uuid]); + this.processNextTask(); + cb(null); + }else cb(err); + }); + }else; // cb is called by find on error + }else cb(err); + }); + } + + // Restarts (puts back into QUEUED state) + // a task that is either in CANCELED or FAILED state. + // When options is set, the task's options are overriden + restart(uuid, options, cb){ + let task = this.find(uuid, cb); + if (task){ + task.restart(options, err => { + if (!err) this.processNextTask(); + cb(err); + }); + } + } + + // Finds a task by its UUID string. + find(uuid, cb){ + let task = this.tasks[uuid]; + if (!task && cb) cb(new Error(`${uuid} not found`)); + return task; + } + + // Serializes the list of tasks and saves it + // to disk + dumpTaskList(done){ + let output = []; + + for (let uuid in this.tasks){ + output.push(this.tasks[uuid].serialize()); + } + + fs.writeFile(TASKS_DUMP_FILE, JSON.stringify(output), err => { + if (err) logger.error(`Could not dump tasks: ${err.message}`); + else logger.debug("Dumped tasks list."); + if (done !== undefined) done(); + }); + } + + getQueueCount(){ + let count = 0; + for (let uuid in this.tasks){ + let task = this.tasks[uuid]; + + if ([statusCodes.QUEUED, + statusCodes.RUNNING].indexOf(task.status.code) !== -1){ + count++; + } + } + return count; + } }; diff --git a/libs/logger.js b/libs/logger.js index 5f5aad83..28f356ec 100644 --- a/libs/logger.js +++ b/libs/logger.js @@ -25,15 +25,15 @@ let path = require('path'); // Set up logging // Configure custom File transport to write plain text messages let logPath = ( config.logger.logDirectory ? - config.logger.logDirectory : - path.join(__dirname, "..") ); + config.logger.logDirectory : + path.join(__dirname, "..") ); // Check that log file directory can be written to try { - fs.accessSync(logPath, fs.W_OK); + fs.accessSync(logPath, fs.W_OK); } catch (e) { - console.log( "Log directory '" + logPath + "' cannot be written to" ); - throw e; + console.log( "Log directory '" + logPath + "' cannot be written to" ); + throw e; } logPath += path.sep; logPath += config.instance + ".log"; @@ -44,16 +44,16 @@ let logger = new (winston.Logger)({ ] }); logger.add(winston.transports.File, { - filename: logPath, // Write to projectname.log - json: false, // Write in plain text, not JSON - maxsize: config.logger.maxFileSize, // Max size of each file - maxFiles: config.logger.maxFiles, // Max number of files - level: config.logger.level // Level of log messages - }); + filename: logPath, // Write to projectname.log + json: false, // Write in plain text, not JSON + maxsize: config.logger.maxFileSize, // Max size of each file + maxFiles: config.logger.maxFiles, // Max number of files + level: config.logger.level // Level of log messages + }); if (config.deamon){ - // Console transport is no use to us when running as a daemon - logger.remove(winston.transports.Console); + // Console transport is no use to us when running as a daemon + logger.remove(winston.transports.Console); } module.exports = logger; \ No newline at end of file diff --git a/libs/odmInfo.js b/libs/odmInfo.js index fea5398d..ff76b76b 100644 --- a/libs/odmInfo.js +++ b/libs/odmInfo.js @@ -26,279 +26,279 @@ let odmOptions = null; let odmVersion = null; module.exports = { - initialize: function(done){ - async.parallel([ - this.getOptions, - this.getVersion - ], done); - }, - - getVersion: function(done){ - if (odmVersion){ - done(null, odmVersion); - return; - } + initialize: function(done){ + async.parallel([ + this.getOptions, + this.getVersion + ], done); + }, + + getVersion: function(done){ + if (odmVersion){ + done(null, odmVersion); + return; + } - odmRunner.getVersion(done); - }, + odmRunner.getVersion(done); + }, - getOptions: function(done){ - if (odmOptions){ - done(null, odmOptions); - return; - } + getOptions: function(done){ + if (odmOptions){ + done(null, odmOptions); + return; + } - odmRunner.getJsonOptions((err, json) => { - if (err) done(err); - else{ - odmOptions = []; - for (let option in json){ - // Not all options are useful to the end user - // (num cores can be set programmatically, so can gcpFile, etc.) - if (["-h", "--project-path", "--cmvs-maxImages", "--time", - "--zip-results", "--pmvs-num-cores", - "--start-with", "--gcp", "--end-with", "--images", - "--rerun-all", "--rerun", - "--slam-config", "--video", "--version", "--name"].indexOf(option) !== -1) continue; + odmRunner.getJsonOptions((err, json) => { + if (err) done(err); + else{ + odmOptions = []; + for (let option in json){ + // Not all options are useful to the end user + // (num cores can be set programmatically, so can gcpFile, etc.) + if (["-h", "--project-path", "--cmvs-maxImages", "--time", + "--zip-results", "--pmvs-num-cores", + "--start-with", "--gcp", "--end-with", "--images", + "--rerun-all", "--rerun", + "--slam-config", "--video", "--version", "--name"].indexOf(option) !== -1) continue; - let values = json[option]; + let values = json[option]; - let name = option.replace(/^--/, ""); - let type = ""; - let value = ""; - let help = values.help || ""; - let domain = values.metavar !== undefined ? - values.metavar.replace(/^[<>]/g, "") - .replace(/[<>]$/g, "") - .trim() : - ""; + let name = option.replace(/^--/, ""); + let type = ""; + let value = ""; + let help = values.help || ""; + let domain = values.metavar !== undefined ? + values.metavar.replace(/^[<>]/g, "") + .replace(/[<>]$/g, "") + .trim() : + ""; - switch((values.type || "").trim()){ - case "": - type = "int"; - value = values['default'] !== undefined ? - parseInt(values['default']) : - 0; - break; - case "": - type = "float"; - value = values['default'] !== undefined ? - parseFloat(values['default']) : - 0.0; - break; - default: - type = "string"; - value = values['default'] !== undefined ? - values['default'].trim() : - ""; - } + switch((values.type || "").trim()){ + case "": + type = "int"; + value = values['default'] !== undefined ? + parseInt(values['default']) : + 0; + break; + case "": + type = "float"; + value = values['default'] !== undefined ? + parseFloat(values['default']) : + 0.0; + break; + default: + type = "string"; + value = values['default'] !== undefined ? + values['default'].trim() : + ""; + } - if (values['default'] === "True"){ - type = "bool"; - value = true; - }else if (values['default'] === "False"){ - type = "bool"; - value = false; - } + if (values['default'] === "True"){ + type = "bool"; + value = true; + }else if (values['default'] === "False"){ + type = "bool"; + value = false; + } - // If 'choices' is specified, try to convert it to array - if (values.choices){ - try{ - values.choices = JSON.parse(values.choices.replace(/'/g, '"')); // Convert ' to " - }catch(e){ - logger.warn(`Cannot parse choices: ${values.choices}`); - } - } + // If 'choices' is specified, try to convert it to array + if (values.choices){ + try{ + values.choices = JSON.parse(values.choices.replace(/'/g, '"')); // Convert ' to " + }catch(e){ + logger.warn(`Cannot parse choices: ${values.choices}`); + } + } - // In the end, all values must be converted back - // to strings (per OpenAPI spec which doesn't allow mixed types) - value = String(value); + // In the end, all values must be converted back + // to strings (per OpenAPI spec which doesn't allow mixed types) + value = String(value); - if (Array.isArray(values.choices)){ - type = "enum"; - domain = values.choices; + if (Array.isArray(values.choices)){ + type = "enum"; + domain = values.choices; - // Make sure that the default value - // is in the list of choices - if (domain.indexOf(value) === -1) domain.unshift(value); - } + // Make sure that the default value + // is in the list of choices + if (domain.indexOf(value) === -1) domain.unshift(value); + } - help = help.replace(/\%\(default\)s/g, value); + help = help.replace(/\%\(default\)s/g, value); - odmOptions.push({ - name, type, value, domain, help - }); - } - done(null, odmOptions); - } - }); - }, + odmOptions.push({ + name, type, value, domain, help + }); + } + done(null, odmOptions); + } + }); + }, - // Checks that the options (as received from the rest endpoint) - // Are valid and within proper ranges. - // The result of filtering is passed back via callback - // @param options[] - filterOptions: function(options, done){ - assert(odmOptions !== null, "odmOptions is not set. Have you initialized odmOptions properly?"); + // Checks that the options (as received from the rest endpoint) + // Are valid and within proper ranges. + // The result of filtering is passed back via callback + // @param options[] + filterOptions: function(options, done){ + assert(odmOptions !== null, "odmOptions is not set. Have you initialized odmOptions properly?"); - try{ - if (typeof options === "string") options = JSON.parse(options); - if (!Array.isArray(options)) options = []; - - let result = []; - let errors = []; - let addError = function(opt, descr){ - errors.push({ - name: opt.name, - error: descr - }); - }; + try{ + if (typeof options === "string") options = JSON.parse(options); + if (!Array.isArray(options)) options = []; + + let result = []; + let errors = []; + let addError = function(opt, descr){ + errors.push({ + name: opt.name, + error: descr + }); + }; - let typeConversion = { - 'float': Number.parseFloat, - 'int': Number.parseInt, - 'bool': function(value){ - if (value === 'true' || value === '1') return true; - else if (value === 'false' || value === '0') return false; - else if (typeof value === 'boolean') return value; - else throw new Error(`Cannot convert ${value} to boolean`); - }, - 'string': function(value){ - return value; // No conversion needed - }, - 'path': function(value){ - return value; // No conversion needed - }, - 'enum': function(value){ - return value; // No conversion needed - } - }; - - let domainChecks = [ - { - regex: /^(positive |negative )?(integer|float)$/, - validate: function(matches, value){ - if (matches[1] === 'positive ') return value >= 0; - else if (matches[1] === 'negative ') return value <= 0; - - else if (matches[2] === 'integer') return Number.isInteger(value); - else if (matches[2] === 'float') return Number.isFinite(value); - } - }, - { - regex: /^percent$/, - validate: function(matches, value){ - return value >= 0 && value <= 100; - } - }, - { - regex: /^(float|integer): ([\-\+\.\d]+) <= x <= ([\-\+\.\d]+)$/, - validate: function(matches, value){ - let [str, type, lower, upper] = matches; - let parseFunc = type === 'float' ? parseFloat : parseInt; - lower = parseFunc(lower); - upper = parseFunc(upper); - return value >= lower && value <= upper; - } - }, - { - regex: /^(float|integer) (>=|>|<|<=) ([\-\+\.\d]+)$/, - validate: function(matches, value){ - let [str, type, oper, bound] = matches; - let parseFunc = type === 'float' ? parseFloat : parseInt; - bound = parseFunc(bound); - switch(oper){ - case '>=': - return value >= bound; - case '>': - return value > bound; - case '<=': - return value <= bound; - case '<': - return value < bound; - default: - return false; - } - } - }, - { - regex: /^(string|path)$/, - validate: function(){ - return true; // All strings/paths are fine - } - } - ]; + let typeConversion = { + 'float': Number.parseFloat, + 'int': Number.parseInt, + 'bool': function(value){ + if (value === 'true' || value === '1') return true; + else if (value === 'false' || value === '0') return false; + else if (typeof value === 'boolean') return value; + else throw new Error(`Cannot convert ${value} to boolean`); + }, + 'string': function(value){ + return value; // No conversion needed + }, + 'path': function(value){ + return value; // No conversion needed + }, + 'enum': function(value){ + return value; // No conversion needed + } + }; + + let domainChecks = [ + { + regex: /^(positive |negative )?(integer|float)$/, + validate: function(matches, value){ + if (matches[1] === 'positive ') return value >= 0; + else if (matches[1] === 'negative ') return value <= 0; + + else if (matches[2] === 'integer') return Number.isInteger(value); + else if (matches[2] === 'float') return Number.isFinite(value); + } + }, + { + regex: /^percent$/, + validate: function(matches, value){ + return value >= 0 && value <= 100; + } + }, + { + regex: /^(float|integer): ([\-\+\.\d]+) <= x <= ([\-\+\.\d]+)$/, + validate: function(matches, value){ + let [str, type, lower, upper] = matches; + let parseFunc = type === 'float' ? parseFloat : parseInt; + lower = parseFunc(lower); + upper = parseFunc(upper); + return value >= lower && value <= upper; + } + }, + { + regex: /^(float|integer) (>=|>|<|<=) ([\-\+\.\d]+)$/, + validate: function(matches, value){ + let [str, type, oper, bound] = matches; + let parseFunc = type === 'float' ? parseFloat : parseInt; + bound = parseFunc(bound); + switch(oper){ + case '>=': + return value >= bound; + case '>': + return value > bound; + case '<=': + return value <= bound; + case '<': + return value < bound; + default: + return false; + } + } + }, + { + regex: /^(string|path)$/, + validate: function(){ + return true; // All strings/paths are fine + } + } + ]; - let checkDomain = function(domain, value){ - if (Array.isArray(domain)){ - // Special case for enum checks - if (domain.indexOf(value) === -1) throw new Error(`Invalid value ${value} (not in enum)`); - }else{ - let matches, - dc = domainChecks.find(dc => matches = domain.match(dc.regex)); + let checkDomain = function(domain, value){ + if (Array.isArray(domain)){ + // Special case for enum checks + if (domain.indexOf(value) === -1) throw new Error(`Invalid value ${value} (not in enum)`); + }else{ + let matches, + dc = domainChecks.find(dc => matches = domain.match(dc.regex)); - if (dc){ - if (!dc.validate(matches, value)) throw new Error(`Invalid value ${value} (out of range)`); - }else{ - throw new Error(`Domain value cannot be handled: '${domain}' : '${value}'`); - } - } - }; + if (dc){ + if (!dc.validate(matches, value)) throw new Error(`Invalid value ${value} (out of range)`); + }else{ + throw new Error(`Domain value cannot be handled: '${domain}' : '${value}'`); + } + } + }; - // Scan through all possible options - let maxConcurrencyFound = false; - let maxConcurrencyIsAnOption = false; + // Scan through all possible options + let maxConcurrencyFound = false; + let maxConcurrencyIsAnOption = false; - for (let odmOption of odmOptions){ - if (odmOption.name === 'max-concurrency') maxConcurrencyIsAnOption = true; - - // Was this option selected by the user? - /*jshint loopfunc: true */ - let opt = options.find(o => o.name === odmOption.name); - if (opt){ - try{ - // Convert to proper data type + for (let odmOption of odmOptions){ + if (odmOption.name === 'max-concurrency') maxConcurrencyIsAnOption = true; + + // Was this option selected by the user? + /*jshint loopfunc: true */ + let opt = options.find(o => o.name === odmOption.name); + if (opt){ + try{ + // Convert to proper data type - let value = typeConversion[odmOption.type](opt.value); + let value = typeConversion[odmOption.type](opt.value); - // Domain check - if (odmOption.domain){ - checkDomain(odmOption.domain, value); - } - - // Max concurrency check - if (opt.name === 'max-concurrency'){ - maxConcurrencyFound = true; + // Domain check + if (odmOption.domain){ + checkDomain(odmOption.domain, value); + } + + // Max concurrency check + if (opt.name === 'max-concurrency'){ + maxConcurrencyFound = true; - // Cap - if (config.maxConcurrency){ - value = Math.min(value, config.maxConcurrency); - } - } + // Cap + if (config.maxConcurrency){ + value = Math.min(value, config.maxConcurrency); + } + } - result.push({ - name: odmOption.name, - value: value - }); - }catch(e){ - addError(opt, e.message); - } - } - } + result.push({ + name: odmOption.name, + value: value + }); + }catch(e){ + addError(opt, e.message); + } + } + } - // If no max concurrency was passed by the user - // but our configuration sets a limit, pass it. - if (!maxConcurrencyFound && maxConcurrencyIsAnOption && config.maxConcurrency){ - result.push({ - name: "max-concurrency", - value: config.maxConcurrency - }); - } + // If no max concurrency was passed by the user + // but our configuration sets a limit, pass it. + if (!maxConcurrencyFound && maxConcurrencyIsAnOption && config.maxConcurrency){ + result.push({ + name: "max-concurrency", + value: config.maxConcurrency + }); + } - if (errors.length > 0) done(new Error(JSON.stringify(errors))); - else done(null, result); - }catch(e){ - done(e); - } - } + if (errors.length > 0) done(new Error(JSON.stringify(errors))); + else done(null, result); + }catch(e){ + done(e); + } + } }; \ No newline at end of file diff --git a/libs/odmRunner.js b/libs/odmRunner.js index da298438..6b315d53 100644 --- a/libs/odmRunner.js +++ b/libs/odmRunner.js @@ -25,112 +25,112 @@ let logger = require('./logger'); module.exports = { - run: function(options, projectName, done, outputReceived){ - assert(projectName !== undefined, "projectName must be specified"); - assert(options["project-path"] !== undefined, "project-path must be defined"); - - const command = path.join(config.odm_path, "run.sh"), - params = []; - - for (var name in options){ - let value = options[name]; - - // Skip false booleans - if (value === false) continue; - - params.push("--" + name); - - // We don't specify "--time true" (just "--time") - if (typeof value !== 'boolean'){ - params.push(value); - } - } - - params.push(projectName); - - logger.info(`About to run: ${command} ${params.join(" ")}`); - - if (config.test){ - logger.info("Test mode is on, command will not execute"); - - let outputTestFile = path.join("..", "tests", "odm_output.txt"); - fs.readFile(path.resolve(__dirname, outputTestFile), 'utf8', (err, text) => { - if (!err){ - let lines = text.split("\n"); - lines.forEach(line => outputReceived(line)); - - done(null, 0, null); - }else{ - logger.warn(`Error: ${err.message}`); - done(err); - } - }); - - return; // Skip rest - } - - // Launch - let childProcess = spawn(command, params, {cwd: config.odm_path}); - - childProcess - .on('exit', (code, signal) => done(null, code, signal)) - .on('error', done); - - childProcess.stdout.on('data', chunk => outputReceived(chunk.toString())); - childProcess.stderr.on('data', chunk => outputReceived(chunk.toString())); - - return childProcess; - }, - - getVersion: function(done){ - fs.readFile(path.join(config.odm_path, 'VERSION'), {encoding: 'utf8'}, (err, content) => { - if (err) done(null, "?"); - else done(null, content.split("\n").map(l => l.trim())[0]); - }); - }, - - getJsonOptions: function(done){ - // In test mode, we don't call ODM, - // instead we return a mock - if (config.test){ - let optionsTestFile = path.join("..", "tests", "odm_options.json"); - fs.readFile(path.resolve(__dirname, optionsTestFile), 'utf8', (err, json) => { - if (!err){ - try{ - let options = JSON.parse(json); - done(null, options); - }catch(e){ - logger.warn(`Invalid test options ${optionsTestFile}: ${err.message}`); - done(e); - } - }else{ - logger.warn(`Error: ${err.message}`); - done(err); - } - }); - - return; // Skip rest - } - - // Launch - let childProcess = spawn("python", [path.join(__dirname, "..", "helpers", "odmOptionsToJson.py"), - "--project-path", config.odm_path, "bogusname"]); - let output = []; - - childProcess - .on('exit', (code, signal) => { - try{ - let json = JSON.parse(output.join("")); - done(null, json); - }catch(err){ - done(new Error(`Could not load list of options from OpenDroneMap. Is OpenDroneMap installed in ${config.odm_path}? Make sure that OpenDroneMap is installed and that --odm_path is set properly: ${err.message}`)); - } - }) - .on('error', done); - - let processOutput = chunk => output.push(chunk.toString()); - - childProcess.stdout.on('data', processOutput); - childProcess.stderr.on('data', processOutput); - } + run: function(options, projectName, done, outputReceived){ + assert(projectName !== undefined, "projectName must be specified"); + assert(options["project-path"] !== undefined, "project-path must be defined"); + + const command = path.join(config.odm_path, "run.sh"), + params = []; + + for (var name in options){ + let value = options[name]; + + // Skip false booleans + if (value === false) continue; + + params.push("--" + name); + + // We don't specify "--time true" (just "--time") + if (typeof value !== 'boolean'){ + params.push(value); + } + } + + params.push(projectName); + + logger.info(`About to run: ${command} ${params.join(" ")}`); + + if (config.test){ + logger.info("Test mode is on, command will not execute"); + + let outputTestFile = path.join("..", "tests", "odm_output.txt"); + fs.readFile(path.resolve(__dirname, outputTestFile), 'utf8', (err, text) => { + if (!err){ + let lines = text.split("\n"); + lines.forEach(line => outputReceived(line)); + + done(null, 0, null); + }else{ + logger.warn(`Error: ${err.message}`); + done(err); + } + }); + + return; // Skip rest + } + + // Launch + let childProcess = spawn(command, params, {cwd: config.odm_path}); + + childProcess + .on('exit', (code, signal) => done(null, code, signal)) + .on('error', done); + + childProcess.stdout.on('data', chunk => outputReceived(chunk.toString())); + childProcess.stderr.on('data', chunk => outputReceived(chunk.toString())); + + return childProcess; + }, + + getVersion: function(done){ + fs.readFile(path.join(config.odm_path, 'VERSION'), {encoding: 'utf8'}, (err, content) => { + if (err) done(null, "?"); + else done(null, content.split("\n").map(l => l.trim())[0]); + }); + }, + + getJsonOptions: function(done){ + // In test mode, we don't call ODM, + // instead we return a mock + if (config.test){ + let optionsTestFile = path.join("..", "tests", "odm_options.json"); + fs.readFile(path.resolve(__dirname, optionsTestFile), 'utf8', (err, json) => { + if (!err){ + try{ + let options = JSON.parse(json); + done(null, options); + }catch(e){ + logger.warn(`Invalid test options ${optionsTestFile}: ${err.message}`); + done(e); + } + }else{ + logger.warn(`Error: ${err.message}`); + done(err); + } + }); + + return; // Skip rest + } + + // Launch + let childProcess = spawn("python", [path.join(__dirname, "..", "helpers", "odmOptionsToJson.py"), + "--project-path", config.odm_path, "bogusname"]); + let output = []; + + childProcess + .on('exit', (code, signal) => { + try{ + let json = JSON.parse(output.join("")); + done(null, json); + }catch(err){ + done(new Error(`Could not load list of options from OpenDroneMap. Is OpenDroneMap installed in ${config.odm_path}? Make sure that OpenDroneMap is installed and that --odm_path is set properly: ${err.message}`)); + } + }) + .on('error', done); + + let processOutput = chunk => output.push(chunk.toString()); + + childProcess.stdout.on('data', processOutput); + childProcess.stderr.on('data', processOutput); + } }; diff --git a/libs/processRunner.js b/libs/processRunner.js index 2a4fbd78..155118ec 100644 --- a/libs/processRunner.js +++ b/libs/processRunner.js @@ -25,56 +25,56 @@ let logger = require('./logger'); function makeRunner(command, args, requiredOptions = [], outputTestFile = null){ - return function(options, done, outputReceived){ - for (let requiredOption of requiredOptions){ - assert(options[requiredOption] !== undefined, `${requiredOption} must be defined`); - } + return function(options, done, outputReceived){ + for (let requiredOption of requiredOptions){ + assert(options[requiredOption] !== undefined, `${requiredOption} must be defined`); + } - let commandArgs = args; - if (typeof commandArgs === 'function') commandArgs = commandArgs(options); + let commandArgs = args; + if (typeof commandArgs === 'function') commandArgs = commandArgs(options); - logger.info(`About to run: ${command} ${commandArgs.join(" ")}`); + logger.info(`About to run: ${command} ${commandArgs.join(" ")}`); - if (config.test){ - logger.info("Test mode is on, command will not execute"); + if (config.test){ + logger.info("Test mode is on, command will not execute"); - if (outputTestFile){ - fs.readFile(path.resolve(__dirname, outputTestFile), 'utf8', (err, text) => { - if (!err){ - let lines = text.split("\n"); - lines.forEach(line => outputReceived(line)); - - done(null, 0, null); - }else{ - logger.warn(`Error: ${err.message}`); - done(err); - } - }); - }else{ - done(null, 0, null); - } + if (outputTestFile){ + fs.readFile(path.resolve(__dirname, outputTestFile), 'utf8', (err, text) => { + if (!err){ + let lines = text.split("\n"); + lines.forEach(line => outputReceived(line)); + + done(null, 0, null); + }else{ + logger.warn(`Error: ${err.message}`); + done(err); + } + }); + }else{ + done(null, 0, null); + } - return;// Skip rest - } + return;// Skip rest + } - // Launch - let childProcess = spawn(command, commandArgs); + // Launch + let childProcess = spawn(command, commandArgs); - childProcess - .on('exit', (code, signal) => done(null, code, signal)) - .on('error', done); + childProcess + .on('exit', (code, signal) => done(null, code, signal)) + .on('error', done); - childProcess.stdout.on('data', chunk => outputReceived(chunk.toString())); - childProcess.stderr.on('data', chunk => outputReceived(chunk.toString())); + childProcess.stdout.on('data', chunk => outputReceived(chunk.toString())); + childProcess.stderr.on('data', chunk => outputReceived(chunk.toString())); - return childProcess; - }; + return childProcess; + }; } module.exports = { - runPostProcessingScript: makeRunner(path.join(__dirname, "..", "scripts", "postprocess.sh"), - function(options){ - return [options.projectFolderPath]; - }, - ["projectFolderPath"]) + runPostProcessingScript: makeRunner(path.join(__dirname, "..", "scripts", "postprocess.sh"), + function(options){ + return [options.projectFolderPath]; + }, + ["projectFolderPath"]) }; diff --git a/libs/statusCodes.js b/libs/statusCodes.js index 5572a5ce..82af6e4a 100644 --- a/libs/statusCodes.js +++ b/libs/statusCodes.js @@ -17,9 +17,9 @@ along with this program. If not, see . */ "use strict"; module.exports = { - QUEUED: 10, - RUNNING: 20, - FAILED: 30, - COMPLETED: 40, - CANCELED: 50 + QUEUED: 10, + RUNNING: 20, + FAILED: 30, + COMPLETED: 40, + CANCELED: 50 }; \ No newline at end of file diff --git a/libs/utils.js b/libs/utils.js index 492771e1..1bb4563c 100644 --- a/libs/utils.js +++ b/libs/utils.js @@ -1,17 +1,17 @@ "use strict"; module.exports = { - get: function(scope, prop, defaultValue){ - let parts = prop.split("."); - let current = scope; - for (let i = 0; i < parts.length; i++){ - if (current[parts[i]] !== undefined && i < parts.length - 1){ - current = current[parts[i]]; - }else if (current[parts[i]] !== undefined && i < parts.length){ - return current[parts[i]]; - }else{ - return defaultValue; - } - } - return defaultValue; - } + get: function(scope, prop, defaultValue){ + let parts = prop.split("."); + let current = scope; + for (let i = 0; i < parts.length; i++){ + if (current[parts[i]] !== undefined && i < parts.length - 1){ + current = current[parts[i]]; + }else if (current[parts[i]] !== undefined && i < parts.length){ + return current[parts[i]]; + }else{ + return defaultValue; + } + } + return defaultValue; + } }; \ No newline at end of file diff --git a/package.json b/package.json index 5b9cea72..d2595ff5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-opendronemap", - "version": "1.3.0", + "version": "1.3.1", "description": "REST API to access ODM", "main": "index.js", "scripts": { diff --git a/public/index.html b/public/index.html index bd6ff7e0..cb1839f6 100644 --- a/public/index.html +++ b/public/index.html @@ -44,16 +44,14 @@

New Task

-
- - +
+
+
-
+
@@ -65,16 +63,29 @@

New Task

- +
+ +
+ +
+ +
+ + + +

+
+

- +

-

diff --git a/public/js/main.js b/public/js/main.js index 3b2adb20..04b28fde 100644 --- a/public/js/main.js +++ b/public/js/main.js @@ -299,6 +299,7 @@ $(function() { name: $("#taskName").val(), zipurl: $("#zipurl").val(), webhook: $("#webhook").val(), + skipPostProcessing: !$("#doPostProcessing").prop('checked'), options: JSON.stringify(optionsModel.getUserOptions()) }; } @@ -326,6 +327,10 @@ $(function() { $("#webhook").val(''); }); + $('#resetDoPostProcessing').on('click', function(){ + $("#doPostProcessing").prop('checked', false); + }); + // zip file control $('#btnShowImport').on('click', function(e){ e.preventDefault();