Skip to content

Commit

Permalink
Add support for file upload in /nodes api
Browse files Browse the repository at this point in the history
  • Loading branch information
@knolleary
knolleary committed Aug 13, 2020
1 parent bcd85b1 commit 6f1ed76
Show file tree
Hide file tree
Showing 10 changed files with 185 additions and 17 deletions.
1 change: 1 addition & 0 deletions package.json
View file
Expand Up @@ -71,6 +71,7 @@
"raw-body": "2.4.1",
"request": "2.88.0",
"semver": "6.3.0",
"tar": "6.0.2",
"uglify-js": "3.10.0",
"when": "3.7.8",
"ws": "6.2.1",
Expand Down
9 changes: 8 additions & 1 deletion packages/node_modules/@node-red/editor-api/lib/admin/index.js
View file
Expand Up @@ -49,7 +49,14 @@ module.exports = {

// Nodes
adminApp.get("/nodes",needsPermission("nodes.read"),nodes.getAll,apiUtil.errorHandler);
adminApp.post("/nodes",needsPermission("nodes.write"),nodes.post,apiUtil.errorHandler);

if (!settings.editorTheme || !settings.editorTheme.palette || settings.editorTheme.palette.upload !== false) {
const multer = require('multer');
const upload = multer({ storage: multer.memoryStorage() });
adminApp.post("/nodes",needsPermission("nodes.write"),upload.single("tarball"),nodes.post,apiUtil.errorHandler);
} else {
adminApp.post("/nodes",needsPermission("nodes.write"),nodes.post,apiUtil.errorHandler);
}
adminApp.get(/^\/nodes\/messages/,needsPermission("nodes.read"),nodes.getModuleCatalogs,apiUtil.errorHandler);
adminApp.get(/^\/nodes\/((@[^\/]+\/)?[^\/]+\/[^\/]+)\/messages/,needsPermission("nodes.read"),nodes.getModuleCatalog,apiUtil.errorHandler);
adminApp.get(/^\/nodes\/((@[^\/]+\/)?[^\/]+)$/,needsPermission("nodes.read"),nodes.getModule,apiUtil.errorHandler);
Expand Down
10 changes: 10 additions & 0 deletions packages/node_modules/@node-red/editor-api/lib/admin/nodes.js
View file
Expand Up @@ -45,8 +45,18 @@ module.exports = {
module: req.body.module,
version: req.body.version,
url: req.body.url,
tarball: undefined,
req: apiUtils.getRequestLogObject(req)
}
if (!runtimeAPI.settings.editorTheme || !runtimeAPI.settings.editorTheme.palette || runtimeAPI.settings.editorTheme.palette.upload !== false) {
if (req.file) {
opts.tarball = {
name: req.file.originalname,
size: req.file.size,
buffer: req.file.buffer
}
}
}
runtimeAPI.nodes.addModule(opts).then(function(info) {
res.json(info);
}).catch(function(err) {
Expand Down
1 change: 1 addition & 0 deletions packages/node_modules/@node-red/editor-api/package.json
View file
Expand Up @@ -26,6 +26,7 @@
"express": "4.17.1",
"memorystore": "1.6.2",
"mime": "2.4.6",
"multer": "1.4.2",
"mustache": "4.0.1",
"oauth2orize": "1.11.0",
"passport-http-bearer": "1.0.1",
Expand Down
116 changes: 108 additions & 8 deletions packages/node_modules/@node-red/registry/lib/installer.js
View file
Expand Up @@ -16,7 +16,9 @@


var path = require("path");
var fs = require("fs");
var os = require("os");
var fs = require("fs-extra");
var tar = require("tar");

var registry = require("./registry");
var library = require("./library");
Expand All @@ -30,9 +32,10 @@ var npmCommand = process.platform === 'win32' ? 'npm.cmd' : 'npm';
var paletteEditorEnabled = false;

var settings;
var moduleRe = /^(@[^/@]+?[/])?[^/@]+?$/;
var slashRe = process.platform === "win32" ? /\\|[/]/ : /[/]/;
var pkgurlRe = /^(https?|git(|\+https?|\+ssh|\+file)):\/\//;
const moduleRe = /^(@[^/@]+?[/])?[^/@]+?$/;
const slashRe = process.platform === "win32" ? /\\|[/]/ : /[/]/;
const pkgurlRe = /^(https?|git(|\+https?|\+ssh|\+file)):\/\//;
const localtgzRe = /^\/.+tgz$/;

function init(runtime) {
events = runtime.events;
Expand All @@ -45,12 +48,14 @@ var activePromise = Promise.resolve();

function checkModulePath(folder) {
var moduleName;
var moduleVersion;
var err;
var fullPath = path.resolve(folder);
var packageFile = path.join(fullPath,'package.json');
try {
var pkg = require(packageFile);
moduleName = pkg.name;
moduleVersion = pkg.version;
if (!pkg['node-red']) {
// TODO: nls
err = new Error("Invalid Node-RED module");
Expand All @@ -62,7 +67,10 @@ function checkModulePath(folder) {
err.code = 404;
throw err;
}
return moduleName;
return {
name: moduleName,
version: moduleVersion
};
}

function checkExistingModule(module,version) {
Expand All @@ -77,7 +85,11 @@ function checkExistingModule(module,version) {
}
return false;
}

function installModule(module,version,url) {
if (Buffer.isBuffer(module)) {
return installTarball(module)
}
module = module || "";
activePromise = activePromise.then(() => {
//TODO: ensure module is 'safe'
Expand All @@ -86,7 +98,7 @@ function installModule(module,version,url) {
var isUpgrade = false;
try {
if (url) {
if (pkgurlRe.test(url)) {
if (pkgurlRe.test(url) || localtgzRe.test(url)) {
// Git remote url or Tarball url - check the valid package url
installName = url;
} else {
Expand All @@ -104,7 +116,8 @@ function installModule(module,version,url) {
} else if (slashRe.test(module)) {
// A path - check if there's a valid package.json
installName = module;
module = checkModulePath(module);
let info = checkModulePath(module);
module = info.name;
} else {
log.warn(log._("server.install.install-failed-name",{name:module}));
e = new Error("Invalid module name");
Expand Down Expand Up @@ -168,7 +181,6 @@ function installModule(module,version,url) {
return activePromise;
}


function reportAddedModules(info) {
//comms.publish("node/added",info.nodes,false);
if (info.nodes.length > 0) {
Expand Down Expand Up @@ -197,6 +209,93 @@ function reportRemovedModules(removedNodes) {
return removedNodes;
}

async function getExistingPackageVersion(moduleName) {
try {
const packageFilename = path.join(settings.userDir || process.env.NODE_RED_HOME || "." , "package.json");
const pkg = await fs.readJson(packageFilename);
if (pkg.dependencies) {
return pkg.dependencies[moduleName];
}
} catch(err) {
}
return null;
}

async function installTarball(tarball) {
// Check this tarball contains a valid node-red module.
// Get its module name/version
const moduleInfo = await getTarballModuleInfo(tarball);

// Write the tarball to <userDir>/nodes/<filename.tgz>
// where the filename is the normalised form based on module name/version
let normalisedModuleName = moduleInfo.name[0] === '@'
? moduleInfo.name.substr(1).replace(/\//g, '-')
: moduleInfo.name
const tarballFile = `${normalisedModuleName}-${moduleInfo.version}.tgz`;
let tarballPath = path.resolve(path.join(settings.userDir || process.env.NODE_RED_HOME || ".", "nodes", tarballFile));

// (from fs-extra - move to writeFile with promise once Node 8 dropped)
await fs.outputFile(tarballPath, tarball);

// Next, need to check to see if this module is listed in `<userDir>/package.json`
let existingVersion = await getExistingPackageVersion(moduleInfo.name);
let existingFile = null;
let isUpdate = false;

// If this is a known module, need to check if there will be an old tarball
// to remove after the install of this one
if (existingVersion) {
// - Known module
if (/^file:nodes\//.test(existingVersion)) {
existingFile = existingVersion.substring(11);
isUpdate = true;
if (tarballFile === existingFile) {
// Edge case: a tar with the same name has bee uploaded.
// Carry on with the install, but don't remove the 'old' file
// as it will have been overwritten by the new one
existingFile = null;
}
}
}

// Install the tgz
return installModule(moduleInfo.name, moduleInfo.version, tarballPath).then(function(info) {
if (existingFile) {
// Remove the old file
return fs.remove(path.resolve(path.join(settings.userDir || process.env.NODE_RED_HOME || ".", "nodes",existingFile))).then(() => info).catch(() => info)
}
return info;
})
}

async function getTarballModuleInfo(tarball) {
const tarballDir = fs.mkdtempSync(path.join(os.tmpdir(),"nr-tarball-"));
const removeExtractedTar = function(done) {
fs.remove(tarballDir, err => {
done();
})
}
return new Promise((resolve,reject) => {
var writeStream = tar.x({
cwd: tarballDir
}).on('error', err => {
reject(err);
}).on('finish', () => {
try {
let moduleInfo = checkModulePath(path.join(tarballDir,"package"));
removeExtractedTar(err => {
resolve(moduleInfo);
})
} catch(err) {
removeExtractedTar(() => {
reject(err);
});
}
});
writeStream.end(tarball);
});
}

function uninstallModule(module) {
activePromise = activePromise.then(() => {
return new Promise((resolve,reject) => {
Expand Down Expand Up @@ -271,6 +370,7 @@ function checkPrereq() {
})
}
}

module.exports = {
init: init,
checkPrereq: checkPrereq,
Expand Down
1 change: 1 addition & 0 deletions packages/node_modules/@node-red/registry/package.json
View file
Expand Up @@ -18,6 +18,7 @@
"dependencies": {
"@node-red/util": "1.2.0-alpha.1",
"semver": "6.3.0",
"tar": "6.0.2",
"uglify-js": "3.10.0",
"when": "3.7.8"
}
Expand Down
32 changes: 32 additions & 0 deletions packages/node_modules/@node-red/runtime/lib/api/nodes.js
View file
Expand Up @@ -159,6 +159,7 @@ var api = module.exports = {
* @param {User} opts.user - the user calling the api
* @param {String} opts.module - the id of the module to install
* @param {String} opts.version - (optional) the version of the module to install
* @param {Object} opts.tarball - (optional) a tarball file to install. Object has properties `name`, `size` and `buffer`.
* @param {String} opts.url - (optional) url to install
* @param {Object} opts.req - the request to log (optional)
* @return {Promise<ModuleInfo>} - the node module info
Expand All @@ -173,6 +174,37 @@ var api = module.exports = {
err.status = 400;
return reject(err);
}
if (opts.tarball) {
if (runtime.settings.editorTheme && runtime.settings.editorTheme.palette && runtime.settings.editorTheme.palette.upload === false) {
runtime.log.audit({event: "nodes.install",tarball:opts.tarball.file,error:"invalid_request"}, opts.req);
var err = new Error("Invalid request");
err.code = "invalid_request";
err.status = 400;
return reject(err);
}
if (opts.module || opts.version || opts.url) {
runtime.log.audit({event: "nodes.install",tarball:opts.tarball.file,module:opts.module,error:"invalid_request"}, opts.req);
var err = new Error("Invalid request");
err.code = "invalid_request";
err.status = 400;
return reject(err);
}
runtime.nodes.installModule(opts.tarball.buffer).then(function(info) {
runtime.log.audit({event: "nodes.install",tarball:opts.tarball.file,module:info.id}, opts.req);
return resolve(info);
}).catch(function(err) {

if (err.code) {
err.status = 400;
runtime.log.audit({event: "nodes.install",module:opts.module,version:opts.version,url:opts.url,error:err.code}, opts.req);
} else {
err.status = 400;
runtime.log.audit({event: "nodes.install",module:opts.module,version:opts.version,url:opts.url,error:err.code||"unexpected_error",message:err.toString()}, opts.req);
}
return reject(err);
})
return;
}
if (opts.module) {
var existingModule = runtime.nodes.getModuleInfo(opts.module);
if (existingModule) {
Expand Down
6 changes: 2 additions & 4 deletions packages/node_modules/@node-red/runtime/lib/nodes/index.js
View file
Expand Up @@ -151,11 +151,9 @@ function reportNodeStateChange(info,enabled) {
}

function installModule(module,version,url) {
var existingModule = registry.getModuleInfo(module);
var isUpgrade = !!existingModule;
return registry.installModule(module,version,url).then(function(info) {
if (isUpgrade) {
events.emit("runtime-event",{id:"node/upgraded",retain:false,payload:{module:module,version:version}});
if (info.pending_version) {
events.emit("runtime-event",{id:"node/upgraded",retain:false,payload:{module:info.name,version:info.pending_version}});
} else {
events.emit("runtime-event",{id:"node/added",retain:false,payload:info.nodes});
}
Expand Down

0 comments on commit 6f1ed76

Please sign in to comment.