From 131fd1ecb628906529170c2daf71a898b743dda6 Mon Sep 17 00:00:00 2001 From: Til Schneider Date: Tue, 27 Aug 2019 05:58:50 +0200 Subject: [PATCH 1/5] Compile messages into a single object literal --- compile.js | 101 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 64 insertions(+), 37 deletions(-) diff --git a/compile.js b/compile.js index e4e4007..24f22e8 100644 --- a/compile.js +++ b/compile.js @@ -14,44 +14,70 @@ function compile(proto) { compile.raw = compileRaw; function compileRaw(proto, options) { - var pre = '\'use strict\'; // code generated by pbf v' + version + '\n'; + var pre = '\'use strict\'; // code generated by pbf v' + version + '\n\n'; var context = buildDefaults(buildContext(proto, null), proto.syntax); return pre + writeContext(context, options || {}); } function writeContext(ctx, options) { - var code = ''; - if (ctx._proto.fields) code += writeMessage(ctx, options); - if (ctx._proto.values) code += writeEnum(ctx, options); + var code, i; + if (!ctx._name) { + // This is the top-level context + code = ''; + for (i = 0; i < ctx._children.length; i++) { + code += writeContext(ctx._children[i], options); + } + return code; + } else if (options.noRead && options.noWrite) { + return '// ' + ctx._fullName + ' ========================================\n\n'; + } else { + if (ctx._root) { + var exportsVar = options.exports || 'exports'; + code = 'var ' + ctx._name + ' = ' + exportsVar + '.' + ctx._name + ' = '; + } else { + code = ctx._indent + ctx._name + ': '; + } - for (var i = 0; i < ctx._children.length; i++) { - code += writeContext(ctx._children[i], options); + if (ctx._proto.fields) { + code += '{\n'; + if (ctx._children.length) { + code += '\n'; + } + code += writeMessage(ctx, options); + for (i = 0; i < ctx._children.length; i++) { + code += ',\n\n'; + code += writeContext(ctx._children[i], options); + } + if (ctx._children.length) { + code += '\n'; + } + code += '\n' + ctx._indent + '}' + (ctx._root ? ';\n\n' : ''); + } else if (ctx._proto.values) { + code += writeEnum(ctx); + code += (ctx._root ? ';' : '') + '\n\n'; + } + + return code; } - return code; } function writeMessage(ctx, options) { - var name = ctx._name; var fields = ctx._proto.fields; var numRepeated = 0; - var code = '\n// ' + name + ' ========================================\n\n'; - - if (!options.noRead || !options.noWrite) { - code += compileExport(ctx, options) + ' {};\n\n'; - } + var code = ''; if (!options.noRead) { - code += name + '.read = function (pbf, end) {\n'; - code += ' return pbf.readFields(' + name + '._readField, ' + compileDest(ctx) + ', end);\n'; - code += '};\n'; - code += name + '._readField = function (tag, obj, pbf) {\n'; + code += ctx._indent + ' read: function (pbf, end) {\n'; + code += ctx._indent + ' return pbf.readFields(' + ctx._fullName + '._readField, ' + compileDest(ctx) + ', end);\n'; + code += ctx._indent + ' },\n'; + code += ctx._indent + ' _readField: function (tag, obj, pbf) {\n'; for (var i = 0; i < fields.length; i++) { var field = fields[i]; var readCode = compileFieldRead(ctx, field); var packed = willSupportPacked(ctx, field); - code += ' ' + (i ? 'else if' : 'if') + + code += ctx._indent + ' ' + (i ? 'else if' : 'if') + ' (tag === ' + field.tag + ') ' + (field.type === 'map' ? ' { ' : '') + ( @@ -66,11 +92,14 @@ function writeMessage(ctx, options) { code += ';' + (field.type === 'map' ? ' }' : '') + '\n'; } - code += '};\n'; + code += ctx._indent + ' }'; + if (!options.noWrite) { + code += ',\n'; + } } if (!options.noWrite) { - code += name + '.write = function (obj, pbf) {\n'; + code += ctx._indent + ' write: function (obj, pbf) {\n'; numRepeated = 0; for (i = 0; i < fields.length; i++) { field = fields[i]; @@ -81,19 +110,14 @@ function writeMessage(ctx, options) { code += getDefaultWriteTest(ctx, field); code += writeCode + ';\n'; } - code += '};\n'; + code += ctx._indent + ' }'; } - return code; -} -function writeEnum(ctx, options) { - return '\n' + compileExport(ctx, options) + ' ' + - JSON.stringify(ctx._proto.values, null, 4) + ';\n'; + return code; } -function compileExport(ctx, options) { - var exportsVar = options.exports || 'exports'; - return (ctx._root ? 'var ' + ctx._name + ' = ' + exportsVar + '.' : '') + ctx._name + ' ='; +function writeEnum(ctx) { + return JSON.stringify(ctx._proto.values, null, 4); } function compileDest(ctx) { @@ -122,8 +146,8 @@ function getType(ctx, field) { function compileFieldRead(ctx, field) { var type = getType(ctx, field); if (type) { - if (type._proto.fields) return type._name + '.read(pbf, pbf.readVarint() + pbf.pos)'; - if (!isEnum(type)) throw new Error('Unexpected type: ' + type._name); + if (type._proto.fields) return type._fullName + '.read(pbf, pbf.readVarint() + pbf.pos)'; + if (!isEnum(type)) throw new Error('Unexpected type: ' + type._fullName); } var fieldType = isEnum(type) ? 'enum' : field.type; @@ -166,9 +190,9 @@ function compileFieldWrite(ctx, field, name) { var type = getType(ctx, field); if (type) { - if (type._proto.fields) return prefix + 'Message(' + field.tag + ', ' + type._name + '.write, ' + name + ')'; + if (type._proto.fields) return prefix + 'Message(' + field.tag + ', ' + type._fullName + '.write, ' + name + ')'; if (type._proto.values) return prefix + 'Varint' + postfix; - throw new Error('Unexpected type: ' + type._name); + throw new Error('Unexpected type: ' + type._fullName); } switch (field.type) { @@ -249,12 +273,15 @@ function buildContext(proto, parent) { if (parent) { parent[proto.name] = obj; - if (parent._name) { + obj._name = proto.name; + if (parent._fullName) { obj._root = false; - obj._name = parent._name + '.' + proto.name; + obj._fullName = parent._fullName + '.' + proto.name; + obj._indent = parent._indent + ' '; } else { obj._root = true; - obj._name = proto.name; + obj._fullName = proto.name; + obj._indent = ''; } } @@ -371,7 +398,7 @@ function buildDefaults(ctx, syntax) { function getDefaultWriteTest(ctx, field) { var def = ctx._defaults[field.name]; var type = getType(ctx, field); - var code = ' if (obj.' + field.name; + var code = ctx._indent + ' if (obj.' + field.name; if (!field.repeated && (!type || !type._proto.fields)) { if (def === undefined || def) { From 881cfcbc6aa956ab3ff60ae7f3ee9756e20d2917 Mon Sep 17 00:00:00 2001 From: Til Schneider Date: Tue, 27 Aug 2019 11:38:13 +0200 Subject: [PATCH 2/5] Add support for generating ES6 and TypeScript code --- README.md | 12 +++-- bin/pbf | 13 ++++- compile.js | 151 ++++++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 159 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 56d5fae..9806022 100644 --- a/README.md +++ b/README.md @@ -281,16 +281,22 @@ Misc methods: For an example of a real-world usage of the library, see [vector-tile-js](https://github.com/mapbox/vector-tile-js). -## Proto Schema to JavaScript +## Proto Schema to JavaScript / TypeScript If installed globally, `pbf` provides a binary that compiles `proto` files into JavaScript modules. Usage: ```bash -$ pbf [--no-write] [--no-read] [--browser] +$ pbf [--no-write] [--no-read] [--browser|--es6|--typescript] ``` The `--no-write` and `--no-read` switches remove corresponding code in the output. -The `--browser` switch makes the module work in browsers instead of Node. + +You can select the JavaScript module type to generate: + +* By default a CommonJS module is generated which works in Node. +* The `--browser` switch generates JavaScript which adds everything to the global namespace which works in a browser. +* The `--es6` switch generates a ES6 module. +* The `--typescript` switch generates a TypeScript module. The resulting module exports each message by name with the following methods: diff --git a/bin/pbf b/bin/pbf index 1064514..78c79c0 100755 --- a/bin/pbf +++ b/bin/pbf @@ -6,12 +6,21 @@ var resolve = require('resolve-protobuf-schema'); var compile = require('../compile'); if (process.argv.length < 3) { - console.error('Usage: pbf [file.proto] [--browser] [--no-read] [--no-write]'); + console.error('Usage: pbf [file.proto] [--browser|--es6|--typescript] [--no-read] [--no-write]'); return; } +var moduleType = 'common-js'; +if (process.argv.indexOf('--browser') >= 0) { + moduleType = 'global'; +} else if (process.argv.indexOf('--es6') >= 0) { + moduleType = 'es6'; +} else if (process.argv.indexOf('--typescript') >= 0) { + moduleType = 'typescript'; +} + var code = compile.raw(resolve.sync(process.argv[2]), { - exports: process.argv.indexOf('--browser') >= 0 ? 'self' : 'exports', + moduleType: moduleType, noRead: process.argv.indexOf('--no-read') >= 0, noWrite: process.argv.indexOf('--no-write') >= 0 }); diff --git a/compile.js b/compile.js index 24f22e8..00d537f 100644 --- a/compile.js +++ b/compile.js @@ -14,9 +14,70 @@ function compile(proto) { compile.raw = compileRaw; function compileRaw(proto, options) { - var pre = '\'use strict\'; // code generated by pbf v' + version + '\n\n'; + options = options || {}; + + var moduleType = options.moduleType; + + var pre = ''; + if (moduleType !== 'es6' && moduleType !== 'typescript') { + pre += '\'use strict\'; '; + } + pre += '// code generated by pbf v' + version + '\n\n'; + if (moduleType === 'typescript') { + pre += 'import Pbf from \'pbf\';\n\n'; + } + var context = buildDefaults(buildContext(proto, null), proto.syntax); - return pre + writeContext(context, options || {}); + return pre + writeTypes(context, options) + writeContext(context, options); +} + +function writeTypes(ctx, options) { + if (options.moduleType !== 'typescript') { + return ''; + } + + var i; + var code = ''; + + var fields = ctx._proto.fields; + if (fields) { + code += 'export interface ' + getTypescriptInterfaceName(ctx) + ' {\n'; + var isOneOfAdded = {}; + for (i = 0; i < fields.length; i++) { + var field = fields[i]; + + var oneOfName = field.oneof; + if (oneOfName && !isOneOfAdded[oneOfName]) { + var oneOfValues = getOneOfValues(fields, oneOfName); + code += ' ' + oneOfName + ': ' + oneOfValues.map(value => JSON.stringify(value)).join(' | ') + ';\n'; + isOneOfAdded[oneOfName] = true; + } + + code += ' ' + field.name + (field.required ? '' : '?') + ': ' + getTypescriptType(ctx, field); + if (field.repeated) { + code += '[]'; + } + code += ';\n'; + } + code += '}\n\n'; + } + + for (i = 0; i < ctx._children.length; i++) { + code += writeTypes(ctx._children[i], options); + } + + return code; +} + +function getOneOfValues(fields, oneOfName) { + var oneOfValues = []; + for (var i = 0; i < fields.length; i++) { + var field = fields[i]; + if (field.oneof === oneOfName) { + oneOfValues.push(field.name); + } + } + return oneOfValues; } function writeContext(ctx, options) { @@ -32,8 +93,12 @@ function writeContext(ctx, options) { return '// ' + ctx._fullName + ' ========================================\n\n'; } else { if (ctx._root) { - var exportsVar = options.exports || 'exports'; - code = 'var ' + ctx._name + ' = ' + exportsVar + '.' + ctx._name + ' = '; + if (options.moduleType === 'es6' || options.moduleType === 'typescript') { + code = 'export const ' + ctx._name + ' = '; + } else { + var exportsVar = (options.moduleType === 'global') ? 'self' : 'exports'; + code = 'var ' + ctx._name + ' = ' + exportsVar + '.' + ctx._name + ' = '; + } } else { code = ctx._indent + ctx._name + ': '; } @@ -63,25 +128,33 @@ function writeContext(ctx, options) { function writeMessage(ctx, options) { var fields = ctx._proto.fields; - var numRepeated = 0; var code = ''; if (!options.noRead) { - code += ctx._indent + ' read: function (pbf, end) {\n'; + code += ctx._indent + ' ' + compileFunctionHead(options, 'read', 'pbf, end', 'pbf: Pbf, end?: number', getTypescriptInterfaceName(ctx)) + ' {\n'; code += ctx._indent + ' return pbf.readFields(' + ctx._fullName + '._readField, ' + compileDest(ctx) + ', end);\n'; code += ctx._indent + ' },\n'; - code += ctx._indent + ' _readField: function (tag, obj, pbf) {\n'; + code += ctx._indent + ' ' + compileFunctionHead(options, '_readField', 'tag, obj, pbf', 'tag: number, obj: any, pbf: Pbf', 'void') + ' {\n'; + var hasVarEntry = false; for (var i = 0; i < fields.length; i++) { var field = fields[i]; var readCode = compileFieldRead(ctx, field); var packed = willSupportPacked(ctx, field); + if (field.type === 'map' && !hasVarEntry) { + code += ctx._indent + ' var entry'; + if (options.moduleType === 'typescript') { + code += ': any'; + } + code += ';\n'; + hasVarEntry = true; + } code += ctx._indent + ' ' + (i ? 'else if' : 'if') + ' (tag === ' + field.tag + ') ' + (field.type === 'map' ? ' { ' : '') + ( - field.type === 'map' ? compileMapRead(readCode, field.name, numRepeated++) : + field.type === 'map' ? compileMapRead(readCode, field.name) : field.repeated && !packed ? 'obj.' + field.name + '.push(' + readCode + ')' : field.repeated && packed ? readCode : 'obj.' + field.name + ' = ' + readCode ); @@ -99,8 +172,8 @@ function writeMessage(ctx, options) { } if (!options.noWrite) { - code += ctx._indent + ' write: function (obj, pbf) {\n'; - numRepeated = 0; + code += ctx._indent + ' ' + compileFunctionHead(options, 'write', 'obj, pbf', 'obj: ' + getTypescriptInterfaceName(ctx) + ', pbf: Pbf', 'void') + ' {\n'; + var numRepeated = 0; for (i = 0; i < fields.length; i++) { field = fields[i]; var writeCode = field.repeated && !isPacked(field) ? @@ -120,6 +193,21 @@ function writeEnum(ctx) { return JSON.stringify(ctx._proto.values, null, 4); } +function compileFunctionHead(options, functionName, params, typedParams, returnType) { + var moduleType = options.moduleType; + var code = functionName; + if (moduleType !== 'es6' && moduleType !== 'typescript') { + code += ': function '; + } + if (moduleType === 'typescript') { + code += '(' + typedParams + '): ' + returnType; + } else { + code += '(' + params + ')'; + } + + return code; +} + function compileDest(ctx) { var props = {}; for (var i = 0; i < ctx._proto.fields.length; i++) { @@ -216,8 +304,8 @@ function compileFieldWrite(ctx, field, name) { } } -function compileMapRead(readCode, name, numRepeated) { - return (numRepeated ? '' : 'var ') + 'entry = ' + readCode + '; obj.' + name + '[entry.key] = entry.value'; +function compileMapRead(readCode, name) { + return 'entry = ' + readCode + '; obj.' + name + '[entry.key] = entry.value'; } function compileRepeatedWrite(ctx, field, numRepeated) { @@ -348,6 +436,45 @@ function willSupportPacked(ctx, field) { return false; } +function getTypescriptInterfaceName(type) { + return 'I' + type._fullName.replace(/\./g, '_'); +} + +function getTypescriptType(ctx, field) { + var type; + switch (field.type) { + case 'float': + case 'double': + case 'uint32': + case 'uint64': + case 'int32': + case 'int64': + case 'sint32': + case 'sint64': + case 'fixed32': + case 'fixed64': + case 'sfixed32': + case 'sfixed64': + return 'number'; + case 'bytes': return 'Uint8Array'; + case 'string': return 'string'; + case 'bool': return 'boolean'; + case 'map': + type = getType(ctx, field); + var keyType = getTypescriptType(ctx, type._proto.fields[0]); + var valueType = getTypescriptType(ctx, type._proto.fields[1]); + return '{ [K in ' + keyType + ']: ' + valueType + ' }'; + default: + type = getType(ctx, field); + if (isEnum(type)) { + return '{ value: number, options: any }'; + } else { + return getTypescriptInterfaceName(type); + } + } +} + + function setPackedOption(ctx, field, syntax) { // No default packed in older protobuf versions if (syntax < 3) return; From b887ea1138b9e22c7aa2bf7f8b00ca2b01b4aa6a Mon Sep 17 00:00:00 2001 From: Til Schneider Date: Fri, 30 Aug 2019 09:46:16 +0200 Subject: [PATCH 3/5] Improve typing of enums --- compile.js | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/compile.js b/compile.js index 00d537f..483360a 100644 --- a/compile.js +++ b/compile.js @@ -62,6 +62,14 @@ function writeTypes(ctx, options) { code += '}\n\n'; } + var values = ctx._proto.values; + if (values) { + code += 'export type ' + getTypescriptEnumKeyTypeName(ctx) + ' = ' + + Object.keys(values).map(function(key) { return JSON.stringify(key); }).join(' | ') + ';\n'; + code += 'export type ' + getTypescriptEnumValueTypeName(ctx) + ' = ' + + Object.values(values).map(function(item) { return item.value; }).join(' | ') + ';\n\n'; + } + for (i = 0; i < ctx._children.length; i++) { code += writeTypes(ctx._children[i], options); } @@ -119,6 +127,9 @@ function writeContext(ctx, options) { code += '\n' + ctx._indent + '}' + (ctx._root ? ';\n\n' : ''); } else if (ctx._proto.values) { code += writeEnum(ctx); + if (options.moduleType === 'typescript') { + code += ' as { [K in ' + getTypescriptEnumKeyTypeName(ctx) + ']: { value: ' + getTypescriptEnumValueTypeName(ctx) + ', options: any } }'; + } code += (ctx._root ? ';' : '') + '\n\n'; } @@ -440,6 +451,14 @@ function getTypescriptInterfaceName(type) { return 'I' + type._fullName.replace(/\./g, '_'); } +function getTypescriptEnumKeyTypeName(type) { + return type._fullName.replace(/\./g, '_') + '_Key'; +} + +function getTypescriptEnumValueTypeName(type) { + return type._fullName.replace(/\./g, '_') + '_Value'; +} + function getTypescriptType(ctx, field) { var type; switch (field.type) { @@ -467,7 +486,7 @@ function getTypescriptType(ctx, field) { default: type = getType(ctx, field); if (isEnum(type)) { - return '{ value: number, options: any }'; + return getTypescriptEnumValueTypeName(type); } else { return getTypescriptInterfaceName(type); } From 534fdd110ff22ff203cc8692808ea02d337cf203 Mon Sep 17 00:00:00 2001 From: Til Schneider Date: Fri, 30 Aug 2019 09:53:37 +0200 Subject: [PATCH 4/5] Avoid two blank lines at the end of generated code --- compile.js | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/compile.js b/compile.js index 483360a..f8db44c 100644 --- a/compile.js +++ b/compile.js @@ -22,9 +22,9 @@ function compileRaw(proto, options) { if (moduleType !== 'es6' && moduleType !== 'typescript') { pre += '\'use strict\'; '; } - pre += '// code generated by pbf v' + version + '\n\n'; + pre += '// code generated by pbf v' + version + '\n'; if (moduleType === 'typescript') { - pre += 'import Pbf from \'pbf\';\n\n'; + pre += '\nimport Pbf from \'pbf\';\n'; } var context = buildDefaults(buildContext(proto, null), proto.syntax); @@ -41,7 +41,7 @@ function writeTypes(ctx, options) { var fields = ctx._proto.fields; if (fields) { - code += 'export interface ' + getTypescriptInterfaceName(ctx) + ' {\n'; + code += '\nexport interface ' + getTypescriptInterfaceName(ctx) + ' {\n'; var isOneOfAdded = {}; for (i = 0; i < fields.length; i++) { var field = fields[i]; @@ -59,15 +59,15 @@ function writeTypes(ctx, options) { } code += ';\n'; } - code += '}\n\n'; + code += '}\n'; } var values = ctx._proto.values; if (values) { - code += 'export type ' + getTypescriptEnumKeyTypeName(ctx) + ' = ' + + code += '\nexport type ' + getTypescriptEnumKeyTypeName(ctx) + ' = ' + Object.keys(values).map(function(key) { return JSON.stringify(key); }).join(' | ') + ';\n'; code += 'export type ' + getTypescriptEnumValueTypeName(ctx) + ' = ' + - Object.values(values).map(function(item) { return item.value; }).join(' | ') + ';\n\n'; + Object.values(values).map(function(item) { return item.value; }).join(' | ') + ';\n'; } for (i = 0; i < ctx._children.length; i++) { @@ -98,17 +98,18 @@ function writeContext(ctx, options) { } return code; } else if (options.noRead && options.noWrite) { - return '// ' + ctx._fullName + ' ========================================\n\n'; + return '\n// ' + ctx._fullName + ' ========================================\n'; } else { + code = '\n'; if (ctx._root) { if (options.moduleType === 'es6' || options.moduleType === 'typescript') { - code = 'export const ' + ctx._name + ' = '; + code += 'export const ' + ctx._name + ' = '; } else { var exportsVar = (options.moduleType === 'global') ? 'self' : 'exports'; - code = 'var ' + ctx._name + ' = ' + exportsVar + '.' + ctx._name + ' = '; + code += 'var ' + ctx._name + ' = ' + exportsVar + '.' + ctx._name + ' = '; } } else { - code = ctx._indent + ctx._name + ': '; + code += ctx._indent + ctx._name + ': '; } if (ctx._proto.fields) { @@ -118,19 +119,19 @@ function writeContext(ctx, options) { } code += writeMessage(ctx, options); for (i = 0; i < ctx._children.length; i++) { - code += ',\n\n'; + code += ',\n'; code += writeContext(ctx._children[i], options); } if (ctx._children.length) { code += '\n'; } - code += '\n' + ctx._indent + '}' + (ctx._root ? ';\n\n' : ''); + code += '\n' + ctx._indent + '}' + (ctx._root ? ';\n' : ''); } else if (ctx._proto.values) { code += writeEnum(ctx); if (options.moduleType === 'typescript') { code += ' as { [K in ' + getTypescriptEnumKeyTypeName(ctx) + ']: { value: ' + getTypescriptEnumValueTypeName(ctx) + ', options: any } }'; } - code += (ctx._root ? ';' : '') + '\n\n'; + code += (ctx._root ? ';' : '') + '\n'; } return code; From 14addebecd2327185f8489bdd6ceb4c861815940 Mon Sep 17 00:00:00 2001 From: Til Schneider Date: Thu, 7 Nov 2019 14:29:46 +0100 Subject: [PATCH 5/5] Add `tslint:disable` comment to generated TypeScript code --- compile.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile.js b/compile.js index f8db44c..c5180df 100644 --- a/compile.js +++ b/compile.js @@ -24,7 +24,7 @@ function compileRaw(proto, options) { } pre += '// code generated by pbf v' + version + '\n'; if (moduleType === 'typescript') { - pre += '\nimport Pbf from \'pbf\';\n'; + pre += '/* tslint:disable */\n\nimport Pbf from \'pbf\';\n'; } var context = buildDefaults(buildContext(proto, null), proto.syntax);