Skip to content

Commit

Permalink
Detect binary modules even if .node file is missing. Fixes gh-542
Browse files Browse the repository at this point in the history
  • Loading branch information
@rwaldron
rwaldron committed Feb 15, 2016
1 parent c0360c1 commit 32f95fe
Show file tree
Hide file tree
Showing 26 changed files with 527 additions and 416 deletions.
179 changes: 155 additions & 24 deletions lib/tessel/deploy.js
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
// System Objects
var cp = require('child_process');
var path = require('path');
var StringDecoder = require('string_decoder').StringDecoder;
var zlib = require('zlib');

// Third Party Dependencies
Expand Down Expand Up @@ -411,30 +413,146 @@ actions.glob = {
}
};

function logMissingBinaryModuleWarning(name) {
var warning = tags.stripIndent `
Pre-compiled module is missing: ${name}.
This might be caused by any of the following:
1. The binary is platform specific and cannot be compiled for OpenWRT.
2. A pre-compiled binary has not yet been generated for this module.
3. The binary didn't compile correctly for the platform that you're developing on.
It's possible that the binary is Linux-only or even OpenWRT specific,
try npm installing with "--force" and rerun your deployment command.
Please file an issue at https://github.com/tessel/t2-cli/issues/new
`;

logs.warn(warning.trim());
}

actions.resolveBinaryModules = function(opts) {
var cwd = process.cwd();
var target = opts.target || cwd;
var relative = path.relative(cwd, target);
var globRoot = relative || target;
var buildexp = /(?:build\/(Debug|Release|bindings)\/)/;

binaryModulesUsed.clear();

return new Promise((resolve, reject) => {
// Find all modules that include a compiled binary
var pattern = 'node_modules/**/*.node';
var binaries = actions.glob.files(globRoot, [pattern]).map(mPath => {
var patterns = ['node_modules/**/*.node', 'node_modules/**/binding.gyp'];
var binaries = actions.glob.files(globRoot, patterns).reduce((bins, globPath) => {
// Gather information about each found module
var binName = path.basename(mPath);
var modulePath = bindings.getRoot(mPath);
var modulePath = bindings.getRoot(globPath);
var packageJson = require(path.join(globRoot, modulePath, 'package.json'));
var binName = path.basename(globPath);
var buildPath = globPath.replace(path.join(modulePath), '').replace(binName, '');
var buildType = (function() {
var matches = buildPath.match(buildexp);
if (matches && matches.length) {
return matches[1];
}
return 'Release';
}());

if (buildType !== 'Release' && buildType !== 'Debug') {
buildType = 'Release';
}

if (binName.endsWith('.gyp')) {

// Check that there are no matching .node paths in the ALREADY PROCESSED paths
for (var i = 0; i < bins.length; i++) {
if (bins[i].modulePath === modulePath) {

// When the currently resolving binary module path has been found
// in the list of existing binary module paths that have already been
// accounted for, return immediately to continue to processing the
// found binary module path candidates.
//
// An example:
//
// [
// "node_modules/bufferutil/build/Release/bufferutil.node",
// "node_modules/bufferutil/binding.gyp",
// ]
//
// Note, the order will always be .node then .gyp
return bins;
}
}

// If unfound, then we have to dig around in the binding.gyp for the target_name
// to figure out the actual name of the .node file
//
// If you're interested in seeing just how much of a nightmare mess these
// files can be, take a look at this:
//
// https://github.com/nodejs/node-gyp/wiki/%22binding.gyp%22-files-out-in-the-wild
//
var bindingGypPath = path.join(globRoot, modulePath, binName);
var bindingGypData;
var bindingGypJson;

try {
// Sometimes people write really tidy binding.gyp files that are
// actually valid JSON, which is totally awesome!
bindingGypJson = require(bindingGypPath);
} catch (error) {
// ... Other times binding.gyp is an inconsistent mess, but it's still
// a valid Python data structure. So we can spawn a python to read it.
// Sounds gross, but there is no other clear way to do this.
bindingGypData = actions.resolveBinaryModules.readGypFileSync(bindingGypPath);

try {
bindingGypJson = JSON.parse(bindingGypData);
} catch (error) {
// If this module's binding.gyp is missing, broken or otherwise
// unusable, log a message about and move on. There are too
// many failure modes here, no way to recover.

logMissingBinaryModuleWarning(packageJson.name);
return bins;
}
}

return {
if (bindingGypJson && Array.isArray(bindingGypJson.targets)) {
// Anything that can't be covered by this will have to be
// dealt with as we encounter them and as they are reported.
binName = bindingGypJson.targets[0].target_name + '.node';

// Assume the most likely scenario first:
//
// build/Release
// build/Debug
//
buildPath = path.join('build', buildType);

// Unless there is a specific `binary.module_path`.
// Checking this only matters when the glob patterns
// didn't turn up a .node binary.
if (packageJson.binary && packageJson.binary.module_path) {
buildPath = path.normalize(packageJson.binary.module_path);
if (buildPath[0] === '.') {
buildPath = buildPath.slice(1);
}
}
}
}

bins.push({
binName: binName,
buildPath: buildPath,
buildType: buildType,
globPath: globPath,
name: packageJson.name,
modulePath: modulePath,
version: packageJson.version,
path: modulePath,
};
});
});

return bins;
}, []);

if (!binaries.length) {
resolve();
Expand All @@ -445,7 +563,7 @@ actions.resolveBinaryModules = function(opts) {
// for each of them.
var requests = binaries.map(details => {
return new Promise((resolve, reject) => {
var tgz = `${details.name}-${details.version}-Release.tgz`;
var tgz = `${details.name}-${details.version}-${details.buildType}.tgz`;

// Store the name of the path where this might already
// be cached, but will most certainly be cached once
Expand All @@ -466,15 +584,14 @@ actions.resolveBinaryModules = function(opts) {

// If an extraction path already exists locally,
// resolve this entry without any further action.
if (fs.existsSync(path.join(details.extractPath, 'Release'))) {
if (fs.existsSync(details.extractPath)) {
return resolve();
} else {
//
// Request, receive, unzip, extract and resolve
//
var url = urljoin(BINARY_SERVER_ROOT, tgz);


// Make a ~/.tessel/binaries/MODULE-NAME directory
fs.mkdirp(details.extractPath, () => {
// wget the tgz, save as
Expand All @@ -486,15 +603,7 @@ actions.resolveBinaryModules = function(opts) {

gunzip.on('error', function(error) {
if (error.code === 'Z_DATA_ERROR') {
var warning = tags.stripIndent `
Pre-compiled module is missing: ${details.name}.
This might be caused by either:
1. The binary is platform specific and cannot be compiled for OpenWRT.
2. A pre-compiled binary has not yet been generated for this module.
Please file an issue at https://github.com/tessel/t2-cli/issues/new
`;

logs.warn(warning.trim());
logMissingBinaryModuleWarning(details.name);

// Remove from tracked binary modules
binaryModulesUsed.delete(details.name);
Expand Down Expand Up @@ -528,16 +637,38 @@ actions.resolveBinaryModules = function(opts) {
});
};

actions.injectBinaryModules = function(globRoot, tempBundlePath) {
actions.resolveBinaryModules.readGypFileSync = function(gypfile) {
var python = process.env.PYTHON || 'python';
var program = `import ast, json; print json.dumps(ast.literal_eval(open("${gypfile}").read()));`;
var decoder = new StringDecoder('utf8');
var result = cp.spawnSync(python, ['-c', program]);
var output = result.output;

return output.reduce((accum, buffer) => {
if (buffer) {
accum += decoder.write(buffer);
}
return accum;
}, '');
};

actions.injectBinaryModules = function(globRoot, tempBundlePath) {
return new Promise((resolve) => {
// For every binary module in use...
binaryModulesUsed.forEach(details => {
var sourceBinary = path.join(details.extractPath, 'Release', details.binName);
var tempTargetModulePath = path.join(tempBundlePath, details.path);
var tempTargetBinary = path.join(tempTargetModulePath, 'build/Release', details.binName);
// console.log(details);
var buildDir = details.buildPath.replace(path.dirname(details.buildPath), '');
var sourceBinary = path.join(details.extractPath, buildDir, details.binName);
var tempTargetModulePath = path.join(tempBundlePath, details.modulePath);
var tempTargetBinary = path.join(tempTargetModulePath, details.buildPath, details.binName);

fs.copySync(sourceBinary, tempTargetBinary);

// Also ensure that package.json was copied.
fs.copySync(
path.join(globRoot, details.modulePath, 'package.json'),
path.join(tempTargetModulePath, 'package.json')
);
});

// All binary modules have been replaced, resolve.
Expand Down

0 comments on commit 32f95fe

Please sign in to comment.