diff --git a/package.json b/package.json index 025ea1cb8a..de431e01d3 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,7 @@ "raw-body": "2.4.1", "request": "2.88.0", "semver": "6.3.0", + "tar": "6.0.2", "uglify-js": "3.10.0", "when": "3.7.8", "ws": "6.2.1", diff --git a/packages/node_modules/@node-red/editor-api/lib/admin/index.js b/packages/node_modules/@node-red/editor-api/lib/admin/index.js index d982c560db..ff32111f59 100644 --- a/packages/node_modules/@node-red/editor-api/lib/admin/index.js +++ b/packages/node_modules/@node-red/editor-api/lib/admin/index.js @@ -49,7 +49,14 @@ module.exports = { // Nodes adminApp.get("/nodes",needsPermission("nodes.read"),nodes.getAll,apiUtil.errorHandler); - adminApp.post("/nodes",needsPermission("nodes.write"),nodes.post,apiUtil.errorHandler); + + if (!settings.editorTheme || !settings.editorTheme.palette || settings.editorTheme.palette.upload !== false) { + const multer = require('multer'); + const upload = multer({ storage: multer.memoryStorage() }); + adminApp.post("/nodes",needsPermission("nodes.write"),upload.single("tarball"),nodes.post,apiUtil.errorHandler); + } else { + adminApp.post("/nodes",needsPermission("nodes.write"),nodes.post,apiUtil.errorHandler); + } adminApp.get(/^\/nodes\/messages/,needsPermission("nodes.read"),nodes.getModuleCatalogs,apiUtil.errorHandler); adminApp.get(/^\/nodes\/((@[^\/]+\/)?[^\/]+\/[^\/]+)\/messages/,needsPermission("nodes.read"),nodes.getModuleCatalog,apiUtil.errorHandler); adminApp.get(/^\/nodes\/((@[^\/]+\/)?[^\/]+)$/,needsPermission("nodes.read"),nodes.getModule,apiUtil.errorHandler); diff --git a/packages/node_modules/@node-red/editor-api/lib/admin/nodes.js b/packages/node_modules/@node-red/editor-api/lib/admin/nodes.js index b78de9d75d..187bd823f9 100644 --- a/packages/node_modules/@node-red/editor-api/lib/admin/nodes.js +++ b/packages/node_modules/@node-red/editor-api/lib/admin/nodes.js @@ -45,8 +45,18 @@ module.exports = { module: req.body.module, version: req.body.version, url: req.body.url, + tarball: undefined, req: apiUtils.getRequestLogObject(req) } + if (!runtimeAPI.settings.editorTheme || !runtimeAPI.settings.editorTheme.palette || runtimeAPI.settings.editorTheme.palette.upload !== false) { + if (req.file) { + opts.tarball = { + name: req.file.originalname, + size: req.file.size, + buffer: req.file.buffer + } + } + } runtimeAPI.nodes.addModule(opts).then(function(info) { res.json(info); }).catch(function(err) { diff --git a/packages/node_modules/@node-red/editor-api/package.json b/packages/node_modules/@node-red/editor-api/package.json index 5f9c17815d..6a2614a9d1 100644 --- a/packages/node_modules/@node-red/editor-api/package.json +++ b/packages/node_modules/@node-red/editor-api/package.json @@ -26,6 +26,7 @@ "express": "4.17.1", "memorystore": "1.6.2", "mime": "2.4.6", + "multer": "1.4.2", "mustache": "4.0.1", "oauth2orize": "1.11.0", "passport-http-bearer": "1.0.1", diff --git a/packages/node_modules/@node-red/registry/lib/installer.js b/packages/node_modules/@node-red/registry/lib/installer.js index 22fb2a5494..54dcb76cb6 100644 --- a/packages/node_modules/@node-red/registry/lib/installer.js +++ b/packages/node_modules/@node-red/registry/lib/installer.js @@ -16,7 +16,9 @@ var path = require("path"); -var fs = require("fs"); +var os = require("os"); +var fs = require("fs-extra"); +var tar = require("tar"); var registry = require("./registry"); var library = require("./library"); @@ -30,9 +32,10 @@ var npmCommand = process.platform === 'win32' ? 'npm.cmd' : 'npm'; var paletteEditorEnabled = false; var settings; -var moduleRe = /^(@[^/@]+?[/])?[^/@]+?$/; -var slashRe = process.platform === "win32" ? /\\|[/]/ : /[/]/; -var pkgurlRe = /^(https?|git(|\+https?|\+ssh|\+file)):\/\//; +const moduleRe = /^(@[^/@]+?[/])?[^/@]+?$/; +const slashRe = process.platform === "win32" ? /\\|[/]/ : /[/]/; +const pkgurlRe = /^(https?|git(|\+https?|\+ssh|\+file)):\/\//; +const localtgzRe = /^\/.+tgz$/; function init(runtime) { events = runtime.events; @@ -45,12 +48,14 @@ var activePromise = Promise.resolve(); function checkModulePath(folder) { var moduleName; + var moduleVersion; var err; var fullPath = path.resolve(folder); var packageFile = path.join(fullPath,'package.json'); try { var pkg = require(packageFile); moduleName = pkg.name; + moduleVersion = pkg.version; if (!pkg['node-red']) { // TODO: nls err = new Error("Invalid Node-RED module"); @@ -62,7 +67,10 @@ function checkModulePath(folder) { err.code = 404; throw err; } - return moduleName; + return { + name: moduleName, + version: moduleVersion + }; } function checkExistingModule(module,version) { @@ -77,7 +85,11 @@ function checkExistingModule(module,version) { } return false; } + function installModule(module,version,url) { + if (Buffer.isBuffer(module)) { + return installTarball(module) + } module = module || ""; activePromise = activePromise.then(() => { //TODO: ensure module is 'safe' @@ -86,7 +98,7 @@ function installModule(module,version,url) { var isUpgrade = false; try { if (url) { - if (pkgurlRe.test(url)) { + if (pkgurlRe.test(url) || localtgzRe.test(url)) { // Git remote url or Tarball url - check the valid package url installName = url; } else { @@ -104,7 +116,8 @@ function installModule(module,version,url) { } else if (slashRe.test(module)) { // A path - check if there's a valid package.json installName = module; - module = checkModulePath(module); + let info = checkModulePath(module); + module = info.name; } else { log.warn(log._("server.install.install-failed-name",{name:module})); e = new Error("Invalid module name"); @@ -168,7 +181,6 @@ function installModule(module,version,url) { return activePromise; } - function reportAddedModules(info) { //comms.publish("node/added",info.nodes,false); if (info.nodes.length > 0) { @@ -197,6 +209,93 @@ function reportRemovedModules(removedNodes) { return removedNodes; } +async function getExistingPackageVersion(moduleName) { + try { + const packageFilename = path.join(settings.userDir || process.env.NODE_RED_HOME || "." , "package.json"); + const pkg = await fs.readJson(packageFilename); + if (pkg.dependencies) { + return pkg.dependencies[moduleName]; + } + } catch(err) { + } + return null; +} + +async function installTarball(tarball) { + // Check this tarball contains a valid node-red module. + // Get its module name/version + const moduleInfo = await getTarballModuleInfo(tarball); + + // Write the tarball to /nodes/ + // where the filename is the normalised form based on module name/version + let normalisedModuleName = moduleInfo.name[0] === '@' + ? moduleInfo.name.substr(1).replace(/\//g, '-') + : moduleInfo.name + const tarballFile = `${normalisedModuleName}-${moduleInfo.version}.tgz`; + let tarballPath = path.resolve(path.join(settings.userDir || process.env.NODE_RED_HOME || ".", "nodes", tarballFile)); + + // (from fs-extra - move to writeFile with promise once Node 8 dropped) + await fs.outputFile(tarballPath, tarball); + + // Next, need to check to see if this module is listed in `/package.json` + let existingVersion = await getExistingPackageVersion(moduleInfo.name); + let existingFile = null; + let isUpdate = false; + + // If this is a known module, need to check if there will be an old tarball + // to remove after the install of this one + if (existingVersion) { + // - Known module + if (/^file:nodes\//.test(existingVersion)) { + existingFile = existingVersion.substring(11); + isUpdate = true; + if (tarballFile === existingFile) { + // Edge case: a tar with the same name has bee uploaded. + // Carry on with the install, but don't remove the 'old' file + // as it will have been overwritten by the new one + existingFile = null; + } + } + } + + // Install the tgz + return installModule(moduleInfo.name, moduleInfo.version, tarballPath).then(function(info) { + if (existingFile) { + // Remove the old file + return fs.remove(path.resolve(path.join(settings.userDir || process.env.NODE_RED_HOME || ".", "nodes",existingFile))).then(() => info).catch(() => info) + } + return info; + }) +} + +async function getTarballModuleInfo(tarball) { + const tarballDir = fs.mkdtempSync(path.join(os.tmpdir(),"nr-tarball-")); + const removeExtractedTar = function(done) { + fs.remove(tarballDir, err => { + done(); + }) + } + return new Promise((resolve,reject) => { + var writeStream = tar.x({ + cwd: tarballDir + }).on('error', err => { + reject(err); + }).on('finish', () => { + try { + let moduleInfo = checkModulePath(path.join(tarballDir,"package")); + removeExtractedTar(err => { + resolve(moduleInfo); + }) + } catch(err) { + removeExtractedTar(() => { + reject(err); + }); + } + }); + writeStream.end(tarball); + }); +} + function uninstallModule(module) { activePromise = activePromise.then(() => { return new Promise((resolve,reject) => { @@ -271,6 +370,7 @@ function checkPrereq() { }) } } + module.exports = { init: init, checkPrereq: checkPrereq, diff --git a/packages/node_modules/@node-red/registry/package.json b/packages/node_modules/@node-red/registry/package.json index 1ae44df8b4..bda29cc94b 100644 --- a/packages/node_modules/@node-red/registry/package.json +++ b/packages/node_modules/@node-red/registry/package.json @@ -18,6 +18,7 @@ "dependencies": { "@node-red/util": "1.2.0-alpha.1", "semver": "6.3.0", + "tar": "6.0.2", "uglify-js": "3.10.0", "when": "3.7.8" } diff --git a/packages/node_modules/@node-red/runtime/lib/api/nodes.js b/packages/node_modules/@node-red/runtime/lib/api/nodes.js index 7624b0b76c..a06cbed966 100644 --- a/packages/node_modules/@node-red/runtime/lib/api/nodes.js +++ b/packages/node_modules/@node-red/runtime/lib/api/nodes.js @@ -159,6 +159,7 @@ var api = module.exports = { * @param {User} opts.user - the user calling the api * @param {String} opts.module - the id of the module to install * @param {String} opts.version - (optional) the version of the module to install + * @param {Object} opts.tarball - (optional) a tarball file to install. Object has properties `name`, `size` and `buffer`. * @param {String} opts.url - (optional) url to install * @param {Object} opts.req - the request to log (optional) * @return {Promise} - the node module info @@ -173,6 +174,37 @@ var api = module.exports = { err.status = 400; return reject(err); } + if (opts.tarball) { + if (runtime.settings.editorTheme && runtime.settings.editorTheme.palette && runtime.settings.editorTheme.palette.upload === false) { + runtime.log.audit({event: "nodes.install",tarball:opts.tarball.file,error:"invalid_request"}, opts.req); + var err = new Error("Invalid request"); + err.code = "invalid_request"; + err.status = 400; + return reject(err); + } + if (opts.module || opts.version || opts.url) { + runtime.log.audit({event: "nodes.install",tarball:opts.tarball.file,module:opts.module,error:"invalid_request"}, opts.req); + var err = new Error("Invalid request"); + err.code = "invalid_request"; + err.status = 400; + return reject(err); + } + runtime.nodes.installModule(opts.tarball.buffer).then(function(info) { + runtime.log.audit({event: "nodes.install",tarball:opts.tarball.file,module:info.id}, opts.req); + return resolve(info); + }).catch(function(err) { + + if (err.code) { + err.status = 400; + runtime.log.audit({event: "nodes.install",module:opts.module,version:opts.version,url:opts.url,error:err.code}, opts.req); + } else { + err.status = 400; + runtime.log.audit({event: "nodes.install",module:opts.module,version:opts.version,url:opts.url,error:err.code||"unexpected_error",message:err.toString()}, opts.req); + } + return reject(err); + }) + return; + } if (opts.module) { var existingModule = runtime.nodes.getModuleInfo(opts.module); if (existingModule) { diff --git a/packages/node_modules/@node-red/runtime/lib/nodes/index.js b/packages/node_modules/@node-red/runtime/lib/nodes/index.js index 09aebe6ebf..705f10f5d0 100644 --- a/packages/node_modules/@node-red/runtime/lib/nodes/index.js +++ b/packages/node_modules/@node-red/runtime/lib/nodes/index.js @@ -151,11 +151,9 @@ function reportNodeStateChange(info,enabled) { } function installModule(module,version,url) { - var existingModule = registry.getModuleInfo(module); - var isUpgrade = !!existingModule; return registry.installModule(module,version,url).then(function(info) { - if (isUpgrade) { - events.emit("runtime-event",{id:"node/upgraded",retain:false,payload:{module:module,version:version}}); + if (info.pending_version) { + events.emit("runtime-event",{id:"node/upgraded",retain:false,payload:{module:info.name,version:info.pending_version}}); } else { events.emit("runtime-event",{id:"node/added",retain:false,payload:info.nodes}); } diff --git a/test/unit/@node-red/editor-api/lib/admin/nodes_spec.js b/test/unit/@node-red/editor-api/lib/admin/nodes_spec.js index 0d373b8d0e..f921d907df 100644 --- a/test/unit/@node-red/editor-api/lib/admin/nodes_spec.js +++ b/test/unit/@node-red/editor-api/lib/admin/nodes_spec.js @@ -53,6 +53,7 @@ describe("api/admin/nodes", function() { describe('get nodes', function() { it('returns node list', function(done) { nodes.init({ + settings: {}, nodes:{ getNodeList: function() { return Promise.resolve([1,2,3]); @@ -75,6 +76,7 @@ describe("api/admin/nodes", function() { it('returns node configs', function(done) { nodes.init({ + settings: {}, nodes:{ getNodeConfigs: function() { return Promise.resolve(""); @@ -99,6 +101,7 @@ describe("api/admin/nodes", function() { it('returns node module info', function(done) { nodes.init({ + settings: {}, nodes:{ getModuleInfo: function(opts) { return Promise.resolve({"node-red":{name:"node-red"}}[opts.module]); @@ -119,6 +122,7 @@ describe("api/admin/nodes", function() { it('returns 404 for unknown module', function(done) { nodes.init({ + settings: {}, nodes:{ getModuleInfo: function(opts) { var errInstance = new Error("Not Found"); @@ -143,6 +147,7 @@ describe("api/admin/nodes", function() { it('returns individual node info', function(done) { nodes.init({ + settings: {}, nodes:{ getNodeInfo: function(opts) { return Promise.resolve({"node-red/123":{id:"node-red/123"}}[opts.id]); @@ -164,6 +169,7 @@ describe("api/admin/nodes", function() { it('returns individual node configs', function(done) { nodes.init({ + settings: {}, nodes:{ getNodeConfig: function(opts) { return Promise.resolve({"node-red/123":""}[opts.id]); @@ -187,6 +193,7 @@ describe("api/admin/nodes", function() { }); it('returns 404 for unknown node', function(done) { nodes.init({ + settings: {}, nodes:{ getNodeInfo: function(opts) { var errInstance = new Error("Not Found"); @@ -215,6 +222,7 @@ describe("api/admin/nodes", function() { it('installs the module and returns module info', function(done) { var opts; nodes.init({ + settings: {}, nodes:{ addModule: function(_opts) { opts = _opts; @@ -244,6 +252,7 @@ describe("api/admin/nodes", function() { }); it('returns error', function(done) { nodes.init({ + settings: {}, nodes:{ addModule: function(opts) { var errInstance = new Error("Message"); @@ -272,6 +281,7 @@ describe("api/admin/nodes", function() { it('uninstalls the module', function(done) { var opts; nodes.init({ + settings: {}, nodes:{ removeModule: function(_opts) { opts = _opts; @@ -292,6 +302,7 @@ describe("api/admin/nodes", function() { }); it('returns error', function(done) { nodes.init({ + settings: {}, nodes:{ removeModule: function(opts) { var errInstance = new Error("Message"); @@ -319,6 +330,7 @@ describe("api/admin/nodes", function() { describe('enable/disable node set', function() { it('returns 400 for invalid request payload', function(done) { nodes.init({ + settings: {}, nodes:{ setNodeSetState: function(opts) {return Promise.resolve()} } @@ -340,6 +352,7 @@ describe("api/admin/nodes", function() { it('sets node state and returns node info', function(done) { var opts; nodes.init({ + settings: {}, nodes:{ setNodeSetState: function(_opts) { opts = _opts; @@ -368,6 +381,7 @@ describe("api/admin/nodes", function() { describe('enable/disable module' ,function() { it('returns 400 for invalid request payload', function(done) { nodes.init({ + settings: {}, nodes:{ setModuleState: function(opts) {return Promise.resolve()} } @@ -388,6 +402,7 @@ describe("api/admin/nodes", function() { it('sets module state and returns module info', function(done) { var opts; nodes.init({ + settings: {}, nodes:{ setModuleState: function(_opts) { opts = _opts; @@ -416,6 +431,7 @@ describe("api/admin/nodes", function() { describe('get icons', function() { it('returns icon list', function(done) { nodes.init({ + settings: {}, nodes:{ getIconList: function() { return Promise.resolve({module:[1,2,3]}); @@ -440,6 +456,7 @@ describe("api/admin/nodes", function() { describe('get module messages', function() { it('returns message catalog', function(done) { nodes.init({ + settings: {}, nodes:{ getModuleCatalog: function(opts) { return Promise.resolve({a:123}); @@ -459,6 +476,7 @@ describe("api/admin/nodes", function() { }); it('returns all node catalogs', function(done) { nodes.init({ + settings: {}, nodes:{ getModuleCatalogs: function(opts) { return Promise.resolve({a:1}); diff --git a/test/unit/@node-red/registry/lib/installer_spec.js b/test/unit/@node-red/registry/lib/installer_spec.js index 1eb79d7238..ad658fd164 100644 --- a/test/unit/@node-red/registry/lib/installer_spec.js +++ b/test/unit/@node-red/registry/lib/installer_spec.js @@ -18,7 +18,7 @@ var should = require("should"); var sinon = require("sinon"); var when = require("when"); var path = require("path"); -var fs = require('fs'); +var fs = require('fs-extra'); var EventEmitter = require('events'); var NR_TEST_UTILS = require("nr-test-utils"); @@ -36,7 +36,7 @@ describe('nodes/registry/installer', function() { warn: sinon.stub(), info: sinon.stub(), metric: sinon.stub(), - _: function() { return "abc"} + _: function(msg) { return msg } } beforeEach(function() { @@ -70,8 +70,8 @@ describe('nodes/registry/installer', function() { typeRegistry.getModuleInfo.restore(); } - if (require('fs').statSync.restore) { - require('fs').statSync.restore(); + if (fs.statSync.restore) { + fs.statSync.restore(); } });