diff --git a/packages/bson-transpilers/README.md b/packages/bson-transpilers/README.md index f6ad1d5d167..e2c2f8ff53d 100644 --- a/packages/bson-transpilers/README.md +++ b/packages/bson-transpilers/README.md @@ -3,7 +3,7 @@ [![downloads][5]][6] Transpilers for building BSON documents in any language. Current support -provided for `shell` `javascript` and `python` as inputs. `java`, `c#`, `node`, `shell` and `python` as +provided for `shell` `javascript` and `python` as inputs. `java`, `c#`, `node`, `shell`, `python` and `ruby` as outputs. > ⚠️  `shell` output produces code that is compatible only with legacy `mongo` shell not the new `mongosh` shell. See [COMPASS-4930](https://jira.mongodb.org/browse/COMPASS-4930) for some additional context @@ -59,6 +59,57 @@ Any transpiler errors that occur will be thrown. To catch them, wrap the - __error.column:__ If it is a syntax error, will have the column. - __error.symbol:__ If it is a syntax error, will have the symbol associated with the error. + +### State + +The `CodeGenerationVisitor` class manages a global state which is bound to the `argsTemplate` functions. This state is intended to be used as a solution for the `argsTemplate` functions to communicate with the `DriverTemplate` function. For example: + +```yaml +ObjectIdEqualsArgsTemplate: &ObjectIdEqualsArgsTemplate !!js/function > + (_) => { + this.oneLineStatement = "Hello World"; + return ''; + } + +DriverTemplate: &DriverTemplate !!js/function > + (_spec) => { + return this.oneLineStatement; + } +``` + +The output of the driver syntax for this language will be the one-line statement `Hello World`. + +#### DeclarationStore +A more practical use-case of state is to accumulate variable declarations throughout the `argsTemplate` to be rendered by the `DriverTemplate`. That is, the motivation for using `DeclarationStore` is to prepend the driver syntax with variable declarations rather than using non-idiomatic solutions such as closures. + +The `DeclarationStore` class maintains an internal state concerning variable declarations. For example, + +```javascript +// within the args template +(arg) => { + return this.declarations.add("Temp", "objectID", (varName) => { + return [ + `${varName}, err := primitive.ObjectIDFromHex(${arg})`, + 'if err != nil {', + ' log.Fatal(err)', + '}' + ].join('\n') + }) +} +``` + +Note that each use of the same variable name will result in an increment being added to the declaration statement. For example, if the variable name `objectIDForTemp` is used two times the resulting declaration statements will use `objectIDForTemp` for the first declaration and `objectID2ForTemp` for the second declaration. The `add` method returns the incremented variable name, and is therefore what would be expected as the right-hand side of the statement defined by the `argsTemplate` function. + +The instance of the `DeclarationStore` constructed by the transpiler class is passed into the driver, syntax via state, for use: + +```javascript +(spec) => { + const comment = '// some comment' + const client = 'client, err := mongo.Connect(context.Background(), options.Client().ApplyURI(cs.String()))' + return "#{comment}\n\n#{client}\n\n${this.declarations.toString()}" +} +``` + ### Errors There are a few different error classes thrown by `bson-transpilers`, each with their own error code: diff --git a/packages/bson-transpilers/codegeneration/CodeGenerationVisitor.js b/packages/bson-transpilers/codegeneration/CodeGenerationVisitor.js index e9eb807d90f..45b5baf8c36 100644 --- a/packages/bson-transpilers/codegeneration/CodeGenerationVisitor.js +++ b/packages/bson-transpilers/codegeneration/CodeGenerationVisitor.js @@ -10,6 +10,7 @@ const { } = require('../helper/error'); const { removeQuotes } = require('../helper/format'); +const DeclarationStore = require('./DeclarationStore'); /** * Class for code generation. Goes in between ANTLR generated visitor and @@ -24,6 +25,7 @@ module.exports = (ANTLRVisitor) => class CodeGenerationVisitor extends ANTLRVisi super(); this.idiomatic = true; // PUBLIC this.clearImports(); + this.state = { declarations: new DeclarationStore() }; } clearImports() { @@ -53,15 +55,32 @@ module.exports = (ANTLRVisitor) => class CodeGenerationVisitor extends ANTLRVisi return this[rule](ctx); } + returnResultWithDeclarations(ctx) { + let result = this.returnResult(ctx); + if (this.getState().declarations.length() > 0) { + result = `${this.getState().declarations.toString() + '\n\n'}${result}`; + } + return result; + } + /** * PUBLIC: This is the entry point for the compiler. Each visitor must define * an attribute called "startNode". * * @param {ParserRuleContext} ctx + * @param {Boolean} useDeclarations - prepend the result string with declarations * @return {String} */ - start(ctx) { - return this.returnResult(ctx).trim(); + start(ctx, useDeclarations = false) { + return (useDeclarations ? this.returnResultWithDeclarations(ctx) : this.returnResult(ctx)).trim(); + } + + getState() { + return this.state; + } + + clearDeclarations() { + this.getState().declarations.clear(); } /** @@ -330,7 +349,7 @@ module.exports = (ANTLRVisitor) => class CodeGenerationVisitor extends ANTLRVisi } returnFunctionCallLhsRhs(lhs, rhs, lhsType, l) { if (lhsType.argsTemplate) { - rhs = lhsType.argsTemplate(l, ...rhs); + rhs = lhsType.argsTemplate.bind(this.getState())(l, ...rhs); } else { rhs = `(${rhs.join(', ')})`; } @@ -494,7 +513,7 @@ module.exports = (ANTLRVisitor) => class CodeGenerationVisitor extends ANTLRVisi ? lhsType.template() : defaultT; const rhs = lhsType.argsTemplate - ? lhsType.argsTemplate(lhsArg, ...args) + ? lhsType.argsTemplate.bind(this.getState())(lhsArg, ...args) : defaultA; const lhs = skipLhs ? '' : lhsArg; return this.Syntax.new.template @@ -516,7 +535,7 @@ module.exports = (ANTLRVisitor) => class CodeGenerationVisitor extends ANTLRVisi let args = ''; const keysAndValues = this.getKeyValueList(ctx); if (ctx.type.argsTemplate) { - args = ctx.type.argsTemplate( + args = ctx.type.argsTemplate.bind(this.getState())( this.getKeyValueList(ctx).map((k) => { return [this.getKeyStr(k), this.visit(this.getValue(k))]; }), @@ -526,7 +545,7 @@ module.exports = (ANTLRVisitor) => class CodeGenerationVisitor extends ANTLRVisi } ctx.indentDepth--; if (ctx.type.template) { - return ctx.type.template(args, ctx.indentDepth); + return ctx.type.template.bind(this.getState())(args, ctx.indentDepth); } return this.visitChildren(ctx); } @@ -545,7 +564,7 @@ module.exports = (ANTLRVisitor) => class CodeGenerationVisitor extends ANTLRVisi if (ctx.type.argsTemplate) { // NOTE: not currently being used anywhere. args = visitedElements.map((arg, index) => { const last = !visitedElements[index + 1]; - return ctx.type.argsTemplate(arg, ctx.indentDepth, last); + return ctx.type.argsTemplate.bind(this.getState())(arg, ctx.indentDepth, last); }).join(''); } else { args = visitedElements.join(', '); diff --git a/packages/bson-transpilers/codegeneration/DeclarationStore.js b/packages/bson-transpilers/codegeneration/DeclarationStore.js new file mode 100644 index 00000000000..db9289cdf57 --- /dev/null +++ b/packages/bson-transpilers/codegeneration/DeclarationStore.js @@ -0,0 +1,90 @@ +/** + * Stores declarations for use in the DriverTemplate + * + * @returns {object} + */ +class DeclarationStore { + constructor() { + this.clear(); + } + + /** + * Add declarations by templateID + varRoot + declaration combo. Duplications will not be collected, rather the add + * method will return the existing declaration's variable name. + * + * @param {string} templateID - Name/alias of the template the declaration is being made for + * @param {string} varRoot - The root of the variable name to be appended by the occurrence count + * @param {function} declaration - The code block to be prepended to the driver syntax + * @returns {string} the variable name with root and appended count + */ + addVar(templateID, varRoot, declaration) { + // Don't push existing declarations + const current = this.alreadyDeclared(templateID, varRoot, declaration); + if (current !== undefined) { + return current; + } + const varName = this.next(templateID, varRoot); + this.vars[declaration(varName)] = varName; + return varName; + } + + /** + * Add a function to the funcs set + * + * @param {string} fn - String literal of a function + */ + addFunc(fn) { + if (!this.funcs[fn]) { + this.funcs[fn] = true; + } + } + + alreadyDeclared(templateID, varRoot, declaration) { + const existing = this.candidates(templateID, varRoot); + for (var i = 0; i < existing.length; i++) { + const candidate = `${this.varTemplateRoot(templateID, varRoot)}${i > 0 ? i : ''}`; + const current = this.vars[declaration(candidate)]; + if (current !== undefined) { + return current; + } + } + } + + candidates(templateID, varRoot) { + const varTemplateRoot = this.varTemplateRoot(templateID, varRoot); + return Object.values(this.vars).filter(varName => varName.startsWith(varTemplateRoot)); + } + + clear() { + this.vars = {}; + this.funcs = {}; + } + + length() { + return Object.keys(this.vars).length + Object.keys(this.funcs).length; + } + + next(templateID, varRoot) { + const existing = this.candidates(templateID, varRoot); + + // If the data does not exist in the vars, then the count should append nothing to the variable name + const count = existing.length > 0 ? existing.length : ''; + return `${this.varTemplateRoot(templateID, varRoot)}${count}`; + } + + /** + * Stringify the variable declarations + * + * @param {string} sep - Seperator string placed between elements in the resulting string of declarations + * @returns {string} all the declarations as a string seperated by a line-break + */ + toString(sep = '\n\n') { + return [...Object.keys(this.vars), ...Object.keys(this.funcs)].join(sep); + } + + varTemplateRoot(templateID, varRoot) { + return `${varRoot}${templateID}`; + } +} + +module.exports = DeclarationStore; diff --git a/packages/bson-transpilers/codegeneration/object/Generator.js b/packages/bson-transpilers/codegeneration/object/Generator.js index 803848b62a0..a7ff1424eeb 100644 --- a/packages/bson-transpilers/codegeneration/object/Generator.js +++ b/packages/bson-transpilers/codegeneration/object/Generator.js @@ -92,7 +92,7 @@ module.exports = (Visitor) => class Generator extends Visitor { } if (lhsType && lhsType.argsTemplate) { - return lhsType.argsTemplate(lhs, ...args); + return lhsType.argsTemplate.bind(this.getState())(lhs, ...args); } let expr; @@ -163,7 +163,7 @@ module.exports = (Visitor) => class Generator extends Visitor { return this.Syntax.equality.template(s, op, this.visit(arr[i + 1])); } if (op === 'in' || op === 'notin') { - return this.Syntax.in.template(s, op, this.visit(arr[i + 1])); + return this.Syntax.in.template.bind(this.state)(s, op, this.visit(arr[i + 1])); } throw new BsonTranspilersRuntimeError(`Unrecognized operation ${op}`); }, this.visit(ctx.children[0])); diff --git a/packages/bson-transpilers/codegeneration/python/Visitor.js b/packages/bson-transpilers/codegeneration/python/Visitor.js index 8a539a43650..0cdab797231 100644 --- a/packages/bson-transpilers/codegeneration/python/Visitor.js +++ b/packages/bson-transpilers/codegeneration/python/Visitor.js @@ -171,7 +171,7 @@ module.exports = (CodeGenerationVisitor) => class Visitor extends CodeGeneration if (ctx.type.argsTemplate) { // NOTE: not currently being used anywhere. args = visitedElements.map((arg, index) => { const last = !visitedElements[index + 1]; - return ctx.type.argsTemplate(arg, ctx.indentDepth, last); + return ctx.type.argsTemplate.bind(this.getState())(arg, ctx.indentDepth, last); }); join = ''; } else { @@ -338,7 +338,7 @@ module.exports = (CodeGenerationVisitor) => class Visitor extends CodeGeneration if (op === 'in' || op === 'notin') { skip = true; if (this.Syntax.in) { - return `${str}${this.Syntax.in.template( + return `${str}${this.Syntax.in.template.bind(this.state)( this.visit(arr[i - 1]), op, this.visit(arr[i + 1]))}`; } return `${str} ${this.visit(arr[i - 1])} ${op} ${this.visit(arr[i + 1])}`; diff --git a/packages/bson-transpilers/index.js b/packages/bson-transpilers/index.js index e70045046e2..a589abc6b45 100644 --- a/packages/bson-transpilers/index.js +++ b/packages/bson-transpilers/index.js @@ -133,8 +133,9 @@ const getTranspiler = (loadTree, visitor, generator, symbols) => { idiomatic; if (!driverSyntax) { transpiler.clearImports(); + transpiler.clearDeclarations(); } - return transpiler.start(tree); + return transpiler.start(tree, !driverSyntax); } catch (e) { if (e.code && e.code.includes('BSONTRANSPILERS')) { throw e; @@ -148,6 +149,7 @@ const getTranspiler = (loadTree, visitor, generator, symbols) => { return { compileWithDriver: (input, idiomatic) => { transpiler.clearImports(); + transpiler.clearDeclarations(); const result = {}; Object.keys(input).map((k) => { @@ -171,7 +173,7 @@ const getTranspiler = (loadTree, visitor, generator, symbols) => { 'Generating driver syntax not implemented for current language' ); } - return transpiler.Syntax.driver(result); + return transpiler.Syntax.driver.bind(transpiler.getState())(result); }, compile: compile, getImports: (driverSyntax) => { diff --git a/packages/bson-transpilers/test/declaration-store.test.js b/packages/bson-transpilers/test/declaration-store.test.js new file mode 100644 index 00000000000..be0e0fbe42c --- /dev/null +++ b/packages/bson-transpilers/test/declaration-store.test.js @@ -0,0 +1,231 @@ +const assert = require('assert'); +const DeclarationStore = require('../codegeneration/DeclarationStore'); + +describe('DeclarationStore', () => { + it('adds data using #add', () => { + const ds = new DeclarationStore(); + + ds.addVar('Temp', 'objectID', (varName) => { return `objectId${varName}`; }); + assert.strictEqual(ds.length(), 1); + }); + it('returns incremented variable names given the pre-incremented variable root-name', () => { + const ds = new DeclarationStore(); + + ds.addVar('ForTemp', 'objectID', () => { return 1; }); + assert.strictEqual(ds.next('ForTemp', 'objectID'), 'objectIDForTemp1'); + + ds.addVar('ForTemp', 'objectID', () => { return 2; }); + assert.strictEqual(ds.next('ForTemp', 'objectID'), 'objectIDForTemp2'); + + ds.addVar('ForTemp', 'objectID', () => { return 3; }); + assert.strictEqual(ds.next('ForTemp', 'objectID'), 'objectIDForTemp3'); + }); + it('stringifies multiple variables declarations', () => { + const ds = new DeclarationStore(); + const declaration1 = (varName) => { + return [] + .concat(`${varName}, err := primitive.ObjectIDFromHex()`) + .concat('if err != nil {') + .concat(' log.Fatal(err)') + .concat('}') + .join('\n'); + }; + + const declaration2 = (varName) => { + return [] + .concat(`${varName}, err := primitive.ObjectIDFromHex("5ab901c29ee65f5c8550c5b9")`) + .concat('if err != nil {') + .concat(' log.Fatal(err)') + .concat('}') + .join('\n'); + }; + ds.addVar('ForTemp', 'objectID', declaration1); + ds.addVar('ForTemp', 'objectID', declaration2); + + const expected = [] + .concat('objectIDForTemp, err := primitive.ObjectIDFromHex()') + .concat('if err != nil {') + .concat(' log.Fatal(err)') + .concat('}') + .concat('') + .concat('objectIDForTemp1, err := primitive.ObjectIDFromHex("5ab901c29ee65f5c8550c5b9")') + .concat('if err != nil {') + .concat(' log.Fatal(err)') + .concat('}') + .join('\n'); + assert.strictEqual(ds.toString(), expected); + }); + it('skips defining declarations for multiple of the exact same declaration (1)', () => { + const ds = new DeclarationStore(); + const declaration1 = (varName) => { + return [] + .concat(`${varName}, err := primitive.ObjectIDFromHex()`) + .concat('if err != nil {') + .concat(' log.Fatal(err)') + .concat('}') + .join('\n'); + }; + + const declaration2 = (varName) => { + return [] + .concat(`${varName}, err := primitive.ObjectIDFromHex("5ab901c29ee65f5c8550c5b9")`) + .concat('if err != nil {') + .concat(' log.Fatal(err)') + .concat('}') + .join('\n'); + }; + + const declaration3 = (varName) => { + return [] + .concat(`${varName}, err := primitive.ObjectIDFromHex()`) + .concat('if err != nil {') + .concat(' log.Fatal(err)') + .concat('}') + .join('\n'); + }; + + ds.addVar('ForTemp', 'objectID', declaration1); + ds.addVar('ForTemp', 'objectID', declaration2); + ds.addVar('ForTemp', 'objectID', declaration3); + + const expected = [] + .concat('objectIDForTemp, err := primitive.ObjectIDFromHex()') + .concat('if err != nil {') + .concat(' log.Fatal(err)') + .concat('}') + .concat('') + .concat('objectIDForTemp1, err := primitive.ObjectIDFromHex("5ab901c29ee65f5c8550c5b9")') + .concat('if err != nil {') + .concat(' log.Fatal(err)') + .concat('}') + .join('\n'); + assert.strictEqual(ds.toString(), expected); + }); + it('skips defining declarations for multiple of the exact same declaration (2)', () => { + const ds = new DeclarationStore(); + const declaration1 = (varName) => { + return [] + .concat(`${varName}, err := primitive.ObjectIDFromHex()`) + .concat('if err != nil {') + .concat(' log.Fatal(err)') + .concat('}') + .join('\n'); + }; + + const declaration2 = (varName) => { + return [] + .concat(`${varName}, err := primitive.ObjectIDFromHex("5ab901c29ee65f5c8550c5b9")`) + .concat('if err != nil {') + .concat(' log.Fatal(err)') + .concat('}') + .join('\n'); + }; + + const declaration3 = (varName) => { + return [] + .concat(`${varName}, err := primitive.ObjectIDFromHex("5ab901c29ee65f5c8550c5b9")`) + .concat('if err != nil {') + .concat(' log.Fatal(err)') + .concat('}') + .join('\n'); + }; + + ds.addVar('ForTemp', 'objectID', declaration1); + ds.addVar('ForTemp', 'objectID', declaration2); + ds.addVar('ForTemp', 'objectID', declaration3); + + const expected = [] + .concat('objectIDForTemp, err := primitive.ObjectIDFromHex()') + .concat('if err != nil {') + .concat(' log.Fatal(err)') + .concat('}') + .concat('') + .concat('objectIDForTemp1, err := primitive.ObjectIDFromHex("5ab901c29ee65f5c8550c5b9")') + .concat('if err != nil {') + .concat(' log.Fatal(err)') + .concat('}') + .join('\n'); + assert.strictEqual(ds.toString(), expected); + }); + it('ignores duplications over different variables', () => { + const ds = new DeclarationStore(); + const declaration1 = (varName) => { + return [] + .concat(`${varName}, err := primitive.ObjectIDFromHex()`) + .concat('if err != nil {') + .concat(' log.Fatal(err)') + .concat('}') + .join('\n'); + }; + + const declaration2 = (varName) => { + return [] + .concat(`${varName}, err := primitive.ObjectIDFromHex("5ab901c29ee65f5c8550c5b9")`) + .concat('if err != nil {') + .concat(' log.Fatal(err)') + .concat('}') + .join('\n'); + }; + + const declaration3 = (varName) => { + return [] + .concat(`${varName}, err := primitive.ObjectIDFromHex("5ab901c29ee65f5c8550c5b9")`) + .concat('if err != nil {') + .concat(' log.Fatal(err)') + .concat('}') + .join('\n'); + }; + + ds.addVar('ForTempA', 'objectID', declaration1); + ds.addVar('ForTempA', 'objectID', declaration2); + ds.addVar('ForTempB', 'objectID', declaration3); + + const expected = [] + .concat('objectIDForTempA, err := primitive.ObjectIDFromHex()') + .concat('if err != nil {') + .concat(' log.Fatal(err)') + .concat('}') + .concat('') + .concat('objectIDForTempA1, err := primitive.ObjectIDFromHex("5ab901c29ee65f5c8550c5b9")') + .concat('if err != nil {') + .concat(' log.Fatal(err)') + .concat('}') + .concat('') + .concat('objectIDForTempB, err := primitive.ObjectIDFromHex("5ab901c29ee65f5c8550c5b9")') + .concat('if err != nil {') + .concat(' log.Fatal(err)') + .concat('}') + .join('\n'); + assert.strictEqual(ds.toString(), expected); + }); + it('ignores duplications over different functions', () => { + const ds = new DeclarationStore(); + const declaration1 = 'var x := func() {}'; + const declaration2 = 'var x := func() {}'; + const declaration3 = 'var y := func() {}'; + + ds.addFunc(declaration1); + ds.addFunc(declaration2); + ds.addFunc(declaration3); + + const expected = [] + .concat('var x := func() {}') + .concat('') + .concat('var y := func() {}') + .join('\n'); + assert.strictEqual(ds.toString(), expected); + }); + it('get length of sets', () => { + const ds = new DeclarationStore(); + const declaration1 = 'var x := func() {}'; + const declaration2 = 'var x := func() {}'; + const declaration3 = 'var y := func() {}'; + + ds.addFunc(declaration1); + ds.addFunc(declaration2); + ds.addFunc(declaration3); + ds.addVar('x', 'y', () => 'z'); + + assert.strictEqual(ds.length(), 3); + }); +});