Permalink
Browse files

CLI: Added an experimental --sparse option to limit pbjs output to ac…

…tually referenced types within main files; Other: Added a few more common google types from google/api, see #433
  • Loading branch information...
dcodeIO committed Feb 4, 2017
1 parent d246024 commit 2ddb76b6e93174787a68f68fb28d26b8ece7cc56
@@ -5,17 +5,19 @@ node_js:
- 4.3.2
- 6
- 7
branches:
only: master
script: npm test && npm run bench

env: CXX=g++-4.8
addons:
apt:
sources: ubuntu-toolchain-r-test
packages: g++-4.8
script: npm test && npm run bench
branches:
only: master

matrix:
include:
- node_js: 6
script: set -e; if [ -n "$SAUCE_USERNAME" ]; then npm install zuul@3.11.1 zuul-ngrok@4.0.0; travis_wait npm run zuul; sleep 3; fi
script: set -e; if [ -n "$SAUCE_USERNAME" ]; then npm install zuul@^3.11.1 zuul-ngrok@^4.0.0; travis_wait npm run zuul; sleep 3; fi
- node_js: 6
script: npm run coverage-ci
script: npm install coveralls@^2.11.15; npm run coverage-ci
@@ -31,7 +31,7 @@ exports.main = function(args, callback) {
lint : "l"
},
string: [ "target", "out", "path", "wrap", "root", "lint" ],
boolean: [ "keep-case", "create", "encode", "decode", "verify", "convert", "delimited", "beautify", "comments", "es6" ],
boolean: [ "keep-case", "create", "encode", "decode", "verify", "convert", "delimited", "beautify", "comments", "es6", "sparse" ],
default: {
target : "json",
create : true,
@@ -51,6 +51,9 @@ exports.main = function(args, callback) {
files = argv._,
paths = typeof argv.path === "string" ? [ argv.path ] : argv.path || [];

// protobuf.js package directory contains additional, otherwise non-bundled google types
paths.push(path.relative(process.cwd(), path.join(__dirname, "..")) || ".");

if (!files.length) {
var descs = Object.keys(targets).filter(function(key) { return !targets[key].private; }).map(function(key) {
return " " + util.pad(key, 14, true) + targets[key].description;
@@ -71,6 +74,8 @@ exports.main = function(args, callback) {
"",
" -o, --out Saves to a file instead of writing to stdout.",
"",
" --sparse Exports only those types referenced from a main file (experimental).",
"",
chalk.bold.gray(" Module targets only:"),
"",
" -w, --wrap Specifies the wrapper to use. Also accepts a path to require a custom wrapper.",
@@ -124,17 +129,33 @@ exports.main = function(args, callback) {

var root = new protobuf.Root();

var mainFiles = [];

// Search include paths when resolving imports
root.resolvePath = function pbjsResolvePath(origin, target) {
var filepath = protobuf.util.path.resolve(origin, target);
if (fs.existsSync(filepath))
return filepath;
var normOrigin = protobuf.util.path.normalize(origin),
normTarget = protobuf.util.path.normalize(target);
if (!normOrigin)
mainFiles.push(normTarget);

var resolved = protobuf.util.path.resolve(normOrigin, normTarget, true);
var idx = resolved.lastIndexOf("google/protobuf/");
if (idx > -1) {
var altname = resolved.substring(idx);
if (altname in protobuf.common)
resolved = altname;
}

if (fs.existsSync(resolved))
return resolved;

for (var i = 0; i < paths.length; ++i) {
var ifilepath = protobuf.util.path.resolve(paths[i] + "/", target);
if (fs.existsSync(ifilepath))
return ifilepath;
var iresolved = protobuf.util.path.resolve(paths[i] + "/", target);
if (fs.existsSync(iresolved))
return iresolved;
}
return filepath;

return resolved;
};

// Use es6 syntax if not explicitly specified on the command line and the es6 wrapper is used
@@ -153,19 +174,28 @@ exports.main = function(args, callback) {
});
process.stdin.on("end", function() {
var source = Buffer.concat(data).toString("utf8");
if (source.charAt(0) !== "{") {
protobuf.parse(source, root, parseOptions);
} else {
var json = JSON.parse(source);
root.setOptions(json.options).addJSON(json);
try {
if (source.charAt(0) !== "{") {
protobuf.parse.filename = "-";
protobuf.parse(source, root, parseOptions);
} else {
var json = JSON.parse(source);
root.setOptions(json.options).addJSON(json);
}
callTarget();
} catch (err) {
if (callback)
return callback(err);
throw err;
}
callTarget();
});

// Load from disk
} else {
try {
root.loadSync(files, parseOptions); // sync is deterministic while async is not
root.loadSync(files, parseOptions).resolveAll(); // sync is deterministic while async is not
if (argv.sparse)
sparsify(root);
callTarget();
} catch (err) {
if (callback) {
@@ -176,6 +206,62 @@ exports.main = function(args, callback) {
}
}

function markReferenced(tobj) {
tobj.referenced = true;
// also mark a type's fields and oneofs
if (tobj.fieldsArray)
tobj.fieldsArray.forEach(function(fobj) {
fobj.referenced = true;
});
if (tobj.oneofsArray)
tobj.oneofsArray.forEach(function(oobj) {
oobj.referenced = true;
});
// also mark an extension field's extended type, but not its (other) fields
if (tobj.extensionField)
tobj.extensionField.parent.referenced = true;
}

function sparsify(root) {

// 1. mark directly or indirectly referenced objects
util.traverse(root, function(obj) {
if (!obj.filename)
return;
if (mainFiles.indexOf(obj.filename) > -1)
util.traverseResolved(obj, markReferenced);
});

// 2. empty unreferenced objects
util.traverse(root, function(obj) {
var parent = obj.parent;
if (!parent || obj.referenced) // root or referenced
return;
// remove unreferenced namespaces
if (obj instanceof protobuf.Namespace) {
var hasReferenced = false;
util.traverse(obj, function(iobj) {
if (iobj.referenced)
hasReferenced = true;
});
if (hasReferenced) { // replace with plain namespace if a namespace subclass
if (obj instanceof protobuf.Type || obj instanceof protobuf.Service) {
var robj = new protobuf.Namespace(obj.name, obj.options);
robj.nested = obj.nested;
parent.add(robj);
}
} else // remove completely if nothing inside is referenced
parent.remove(obj);

// remove everything else unreferenced
} else if (!(obj instanceof protobuf.Namespace))
parent.remove(obj);
});

// 3. validate that everything is fine
root.resolveAll();
}

function callTarget() {
target(root, argv, function targetCallback(err, output) {
if (err) {
@@ -33,6 +33,38 @@ exports.requireAll = function requireAll(dirname) {
return all;
};

exports.traverse = function traverse(current, fn) {
fn(current);
if (current.fieldsArray)
current.fieldsArray.forEach(function(field) {
traverse(field, fn);
});
if (current.oneofsArray)
current.oneofsArray.forEach(function(oneof) {
traverse(oneof, fn);
});
if (current.methodsArray)
current.methodsArray.forEach(function(method) {
traverse(method, fn);
});
if (current.nestedArray)
current.nestedArray.forEach(function(nested) {
traverse(nested, fn);
});
};

exports.traverseResolved = function traverseResolved(current, fn) {
fn(current);
if (current.resolvedType)
traverseResolved(current.resolvedType, fn);
if (current.resolvedKeyType)
traverseResolved(current.resolvedKeyType, fn);
if (current.resolvedRequestType)
traverseResolved(current.resolvedRequestType, fn);
if (current.resolvedResponseType)
traverseResolved(current.resolvedResponseType, fn);
};

exports.inspect = function inspect(object, indent) {
if (!object)
return "";
@@ -0,0 +1,83 @@
{
"nested": {
"google": {
"nested": {
"api": {
"nested": {
"http": {
"type": "HttpRule",
"id": 72295728,
"extend": "google.protobuf.MethodOptions"
},
"HttpRule": {
"oneofs": {
"pattern": {
"oneof": [
"get",
"put",
"post",
"delete",
"patch",
"custom"
]
}
},
"fields": {
"get": {
"type": "string",
"id": 2
},
"put": {
"type": "string",
"id": 3
},
"post": {
"type": "string",
"id": 4
},
"delete": {
"type": "string",
"id": 5
},
"patch": {
"type": "string",
"id": 6
},
"custom": {
"type": "CustomHttpPattern",
"id": 8
},
"selector": {
"type": "string",
"id": 1
},
"body": {
"type": "string",
"id": 7
},
"additionalBindings": {
"rule": "repeated",
"type": "HttpRule",
"id": 11
}
}
}
}
},
"protobuf": {
"nested": {
"MethodOptions": {
"fields": {},
"extensions": [
[
1000,
536870911
]
]
}
}
}
}
}
}
}
@@ -0,0 +1,11 @@
syntax = "proto3";

package google.api;

import "google/api/http.proto";
import "google/protobuf/descriptor.proto";

extend google.protobuf.MethodOptions {

HttpRule http = 72295728;
}
Oops, something went wrong.

0 comments on commit 2ddb76b

Please sign in to comment.