Skip to content

Commit

Permalink
Handle subflow modules with their own npm dependencies
Browse files Browse the repository at this point in the history
  • Loading branch information
@knolleary
knolleary committed Nov 25, 2020
1 parent 28184e8 commit 1b33f87
Show file tree
Hide file tree
Showing 16 changed files with 242 additions and 74 deletions.
1 change: 1 addition & 0 deletions packages/node_modules/@node-red/editor-api/lib/admin/nodes.js
View file
Expand Up @@ -60,6 +60,7 @@ module.exports = {
runtimeAPI.nodes.addModule(opts).then(function(info) {
res.json(info);
}).catch(function(err) {
console.log(err.stack);
apiUtils.rejectHandler(req,res,err);
})
},
Expand Down
1 change: 1 addition & 0 deletions packages/node_modules/@node-red/editor-client/src/js/red.js
View file
Expand Up @@ -386,6 +386,7 @@ var RED = (function() {
}
});
RED.comms.subscribe("notification/node/#",function(topic,msg) {
console.log(topic,msg);
var i,m;
var typeList;
var info;
Expand Down
42 changes: 26 additions & 16 deletions packages/node_modules/@node-red/registry/lib/installer.js
View file
Expand Up @@ -73,19 +73,6 @@ function checkModulePath(folder) {
};
}

function checkExistingModule(module,version) {
var info = registry.getModuleInfo(module);
if (info) {
if (!version || info.version === version) {
var err = new Error("Module already loaded");
err.code = "module_already_loaded";
throw err;
}
return true;
}
return false;
}

function installModule(module,version,url) {
if (Buffer.isBuffer(module)) {
return installTarball(module)
Expand All @@ -96,6 +83,7 @@ function installModule(module,version,url) {
return new Promise((resolve,reject) => {
var installName = module;
var isUpgrade = false;
var isExisting = false;
try {
if (url) {
if (pkgurlRe.test(url) || localtgzRe.test(url)) {
Expand Down Expand Up @@ -125,7 +113,20 @@ function installModule(module,version,url) {
reject(e);
return;
}
isUpgrade = checkExistingModule(module,version);
var info = registry.getModuleInfo(module);
if (info) {
if (!info.user) {
log.debug(`Installing existing module: ${module}`)
isExisting = true;
} else if (!version || info.version === version) {
var err = new Error("Module already loaded");
err.code = "module_already_loaded";
throw err;
}
isUpgrade = true;
} else {
isUpgrade = false;
}
} catch(err) {
return reject(err);
}
Expand All @@ -141,7 +142,17 @@ function installModule(module,version,url) {
exec.run(npmCommand,args,{
cwd: installDir
}, true).then(result => {
if (!isUpgrade) {
if (isExisting) {
// This is a module we already have installed as a non-user module.
// That means it was discovered when loading, but was not listed
// in package.json and has been hidden from the editor.
// The user has requested to install this module. Having run
// the npm install above, it will now be listed in package.json.
// Update the registry to mark it as a user module so it will
// be available to the editor.
log.info(log._("server.install.installed",{name:module}));
resolve(require("./registry").setUserInstalled(module,true).then(reportAddedModules));
} else if (!isUpgrade) {
log.info(log._("server.install.installed",{name:module}));
resolve(require("./index").addModule(module).then(reportAddedModules));
} else {
Expand Down Expand Up @@ -182,7 +193,6 @@ function installModule(module,version,url) {
}

function reportAddedModules(info) {
//comms.publish("node/added",info.nodes,false);
if (info.nodes.length > 0) {
log.info(log._("server.added-types"));
for (var i=0;i<info.nodes.length;i++) {
Expand Down
28 changes: 25 additions & 3 deletions packages/node_modules/@node-red/registry/lib/loader.js
View file
Expand Up @@ -310,15 +310,37 @@ function addModule(module) {
throw new Error("Settings unavailable");
}
var nodes = [];
if (registry.getModuleInfo(module)) {
var existingInfo = registry.getModuleInfo(module);
if (existingInfo) {
// TODO: nls
var e = new Error("module_already_loaded");
e.code = "module_already_loaded";
return Promise.reject(e);
}
try {
var moduleFiles = localfilesystem.getModuleFiles(module);
return loadNodeFiles(moduleFiles);
var moduleFiles = {};
var moduleStack = [module];
while(moduleStack.length > 0) {
var moduleToLoad = moduleStack.shift();
var files = localfilesystem.getModuleFiles(moduleToLoad);
if (files[moduleToLoad]) {
moduleFiles[moduleToLoad] = files[moduleToLoad];
if (moduleFiles[moduleToLoad].dependencies) {
log.debug(`Loading dependencies for ${module}`)
for (var i=0; i<moduleFiles[moduleToLoad].dependencies.length; i++) {
var dep = moduleFiles[moduleToLoad].dependencies[i]
if (!registry.getModuleInfo(dep)) {
log.debug(` - load ${dep}`)
moduleStack.push(dep);
} else {
log.debug(` - already loaded ${dep}`)
registry.addModuleDependency(dep,moduleToLoad)
}
}
}
}
}
return loadNodeFiles(moduleFiles).then(() => module)
} catch(err) {
return Promise.reject(err);
}
Expand Down
109 changes: 77 additions & 32 deletions packages/node_modules/@node-red/registry/lib/localfilesystem.js
View file
Expand Up @@ -26,6 +26,7 @@ var i18n = require("@node-red/util").i18n;
var settings;
var disableNodePathScan = false;
var iconFileExtensions = [".png", ".gif", ".svg"];
var packageList = {};

function init(runtime) {
settings = runtime.settings;
Expand Down Expand Up @@ -173,9 +174,17 @@ function scanTreeForNodesModules(moduleName) {
var userDir;

if (settings.userDir) {
packageList = getPackageList();
userDir = path.join(settings.userDir,"node_modules");
results = scanDirForNodesModules(userDir,moduleName);
results.forEach(function(r) { r.local = true; });
results.forEach(function(r) {
// If it was found in <userDir>/node_modules then it is considered
// a local module.
// Also check to see if it is listed in the package.json file as a user-installed
// module. This distinguishes modules installed as a dependency
r.local = true;
r.user = !!packageList[r.package.name];
});
}

if (dir) {
Expand Down Expand Up @@ -275,20 +284,19 @@ function getNodeFiles(disableNodePathScan) {
}
}

var nodeList = {
"node-red": {
name: "node-red",
version: settings.version,
nodes: {},
icons: iconList
}
var nodeList = {};
var coreNodeEntry = {
name: "node-red",
version: settings.version,
nodes: {},
icons: iconList
}
nodeFiles.forEach(function(node) {
nodeList["node-red"].nodes[node.name] = node;
coreNodeEntry.nodes[node.name] = node;
});
if (settings.coreNodesDir) {
var examplesDir = path.join(settings.coreNodesDir,"examples");
nodeList["node-red"].examples = {path: examplesDir};
coreNodeEntry.examples = {path: examplesDir};
}

if (!disableNodePathScan) {
Expand All @@ -297,7 +305,6 @@ function getNodeFiles(disableNodePathScan) {
// Filter the module list to ignore global modules
// that have also been installed locally - allowing the user to
// update a module they may not otherwise be able to touch

moduleFiles.sort(function(A,B) {
if (A.local && !B.local) {
return -1
Expand All @@ -310,7 +317,7 @@ function getNodeFiles(disableNodePathScan) {
moduleFiles = moduleFiles.filter(function(mod) {
var result;
if (!knownModules[mod.package.name]) {
knownModules[mod.package.name] = true;
knownModules[mod.package.name] = mod;
result = true;
} else {
result = false;
Expand All @@ -320,56 +327,77 @@ function getNodeFiles(disableNodePathScan) {
return result;
});

moduleFiles.forEach(function(moduleFile) {
var nodeModuleFiles = getModuleNodeFiles(moduleFile);
nodeList[moduleFile.package.name] = {
name: moduleFile.package.name,
version: moduleFile.package.version,
path: moduleFile.dir,
local: moduleFile.local||false,
nodes: {},
icons: nodeModuleFiles.icons,
examples: nodeModuleFiles.examples
};
if (moduleFile.package['node-red'].version) {
nodeList[moduleFile.package.name].redVersion = moduleFile.package['node-red'].version;
// Do a second pass to check we have all the declared node dependencies
// As this is only done as part of the initial palette load, `knownModules` will
// contain a list of everything discovered during this phase. This means
// we can check for missing dependencies here.
moduleFiles = moduleFiles.filter(function(mod) {
if (Array.isArray(mod.package["node-red"].dependencies)) {
const deps = mod.package["node-red"].dependencies;
const missingDeps = mod.package["node-red"].dependencies.filter(dep => {
if (knownModules[dep]) {
knownModules[dep].usedBy = knownModules[dep].usedBy || [];
knownModules[dep].usedBy.push(mod.package.name)
} else {
return true;
}
})
if (missingDeps.length > 0) {
log.error(`Module: ${mod.package.name} missing dependencies:`);
missingDeps.forEach(m => { log.error(` - ${m}`)});
return false;
}
}
nodeModuleFiles.files.forEach(function(node) {
node.local = moduleFile.local||false;
nodeList[moduleFile.package.name].nodes[node.name] = node;
});
nodeFiles = nodeFiles.concat(nodeModuleFiles.files);
return true;
});
nodeList = convertModuleFileListToObject(moduleFiles);
} else {
// console.log("node path scan disabled");
}
nodeList["node-red"] = coreNodeEntry;
return nodeList;
}

function getModuleFiles(module) {
var nodeList = {};

// Update the package list
var moduleFiles = scanTreeForNodesModules(module);
if (moduleFiles.length === 0) {
var err = new Error(log._("nodes.registry.localfilesystem.module-not-found", {module:module}));
err.code = 'MODULE_NOT_FOUND';
throw err;
}
// Unlike when doing the initial palette load, this call cannot verify the
// dependencies of the new module as it doesn't have visiblity of what
// is in the registry. That will have to be done be the caller in loader.js
return convertModuleFileListToObject(moduleFiles);
}

function convertModuleFileListToObject(moduleFiles) {
const nodeList = {};
moduleFiles.forEach(function(moduleFile) {

var nodeModuleFiles = getModuleNodeFiles(moduleFile);
nodeList[moduleFile.package.name] = {
name: moduleFile.package.name,
version: moduleFile.package.version,
path: moduleFile.dir,
local: moduleFile.local||false,
user: moduleFile.user||false,
nodes: {},
icons: nodeModuleFiles.icons,
examples: nodeModuleFiles.examples
};
if (moduleFile.package['node-red'].version) {
nodeList[moduleFile.package.name].redVersion = moduleFile.package['node-red'].version;
}
if (moduleFile.package['node-red'].dependencies) {
nodeList[moduleFile.package.name].dependencies = moduleFile.package['node-red'].dependencies;
}
if (moduleFile.usedBy) {
nodeList[moduleFile.package.name].usedBy = moduleFile.usedBy;
}
nodeModuleFiles.files.forEach(function(node) {
node.local = moduleFile.local||false;
nodeList[moduleFile.package.name].nodes[node.name] = node;
nodeList[moduleFile.package.name].nodes[node.name].local = moduleFile.local || false;
});
Expand Down Expand Up @@ -400,6 +428,23 @@ function scanIconDir(dir) {
})
return iconList;
}
/**
* Gets the list of modules installed in this runtime as reported by package.json
* Note: these may include non-Node-RED modules
*/
function getPackageList() {
var list = {};
if (settings.userDir) {
try {
var userPackage = path.join(settings.userDir,"package.json");
var pkg = JSON.parse(fs.readFileSync(userPackage,"utf-8"));
return pkg.dependencies;
} catch(err) {
log.error(err);
}
}
return list;
}

module.exports = {
init: init,
Expand Down

0 comments on commit 1b33f87

Please sign in to comment.