feat(export-to-language): Add DeclarationStore to transpiler

packages/bson-transpilers/README.md

@@ -3,7 +3,7 @@
 [![downloads][5]][6]
 
 Transpilers for building BSON documents in any language. Current support
-provided for `shell` `javascript` and `python` as inputs. `java`, `c#`, `node`, `shell` and `python` as
+provided for `shell` `javascript` and `python` as inputs. `java`, `c#`, `node`, `shell`, `python` and `ruby` as
 outputs.
 
 > ⚠️  `shell` output produces code that is compatible only with legacy `mongo` shell not the new `mongosh` shell. See [COMPASS-4930](https://jira.mongodb.org/browse/COMPASS-4930) for some additional context
@@ -59,6 +59,57 @@ Any transpiler errors that occur will be thrown. To catch them, wrap the
 - __error.column:__ If it is a syntax error, will have the column.
 - __error.symbol:__ If it is a syntax error, will have the symbol associated with the error.
 
+
+### State
+
+The `CodeGenerationVisitor` class manages a global state which is bound to the `argsTemplate` functions.  This state is intended to be used as a solution for the `argsTemplate` functions to communicate with the `DriverTemplate` function.  For example:
+
+```yaml
+ObjectIdEqualsArgsTemplate: &ObjectIdEqualsArgsTemplate !!js/function >
+    (_) => {
+        this.oneLineStatement = "Hello World";
+        return '';
+    }
+
+DriverTemplate: &DriverTemplate !!js/function >
+    (_spec) => {
+      return this.oneLineStatement;
+    }
+```
+
+The output of the driver syntax for this language will be the one-line statement `Hello World`.
+
+#### DeclarationStore
+A more practical use-case of state is to accumulate variable declarations throughout the `argsTemplate` to be rendered by the `DriverTemplate`.  That is, the motivation for using `DeclarationStore` is to prepend the driver syntax with variable declarations rather than using non-idiomatic solutions such as closures.
+
+The `DeclarationStore` class maintains an internal state concerning variable declarations.  For example,
+
+```javascript
+// within the args template
+(arg) => {
+  return this.declarations.add("Temp", "objectID", (varName) => {
+    return [
+      `${varName}, err := primitive.ObjectIDFromHex(${arg})`,
+      'if err != nil {',
+      '   log.Fatal(err)',
+      '}'
+    ].join('\n')
+  })
+}
+```
+
+Note that each use of the same variable name will result in an increment being added to the declaration statement. For example, if the variable name `objectIDForTemp` is used two times the resulting declaration statements will use `objectIDForTemp` for the first declaration and `objectID2ForTemp` for the second declaration.  The `add` method returns the incremented variable name, and is therefore what would be expected as the right-hand side of the statement defined by the `argsTemplate` function.
+
+The instance of the `DeclarationStore` constructed by the transpiler class is passed into the driver, syntax via state, for use:
+
+```javascript
+(spec) => {
+  const comment = '// some comment'
+  const client = 'client, err := mongo.Connect(context.Background(), options.Client().ApplyURI(cs.String()))'
+  return "#{comment}\n\n#{client}\n\n${this.declarations.toString()}"
+}
+```
+
 ### Errors
 There are a few different error classes thrown by `bson-transpilers`, each with
 their own error code:

packages/bson-transpilers/codegeneration/CodeGenerationVisitor.js

@@ -10,6 +10,7 @@ const {
 } = require('../helper/error');
 
 const { removeQuotes } = require('../helper/format');
+const DeclarationStore = require('./DeclarationStore');
 
 /**
  * Class for code generation. Goes in between ANTLR generated visitor and
@@ -24,6 +25,7 @@ module.exports = (ANTLRVisitor) => class CodeGenerationVisitor extends ANTLRVisi
     super();
     this.idiomatic = true; // PUBLIC
     this.clearImports();
+    this.state = { declarations: new DeclarationStore() };
   }
 
   clearImports() {
@@ -53,15 +55,32 @@ module.exports = (ANTLRVisitor) => class CodeGenerationVisitor extends ANTLRVisi
     return this[rule](ctx);
   }
 
+  returnResultWithDeclarations(ctx) {
+    let result = this.returnResult(ctx);
+    if (this.getState().declarations.length() > 0) {
+      result = `${this.getState().declarations.toString() + '\n\n'}${result}`;
+    }
+    return result;
+  }
+
   /**
    * PUBLIC: This is the entry point for the compiler. Each visitor must define
    * an attribute called "startNode".
    *
    * @param {ParserRuleContext} ctx
+   * @param {Boolean} useDeclarations - prepend the result string with declarations
    * @return {String}
    */
-  start(ctx) {
-    return this.returnResult(ctx).trim();
+  start(ctx, useDeclarations = false) {
+    return (useDeclarations ? this.returnResultWithDeclarations(ctx) : this.returnResult(ctx)).trim();
+  }
+
+  getState() {
+    return this.state;
+  }
+
+  clearDeclarations() {
+    this.getState().declarations.clear();
   }
 
   /**
@@ -330,7 +349,7 @@ module.exports = (ANTLRVisitor) => class CodeGenerationVisitor extends ANTLRVisi
   }
   returnFunctionCallLhsRhs(lhs, rhs, lhsType, l) {
     if (lhsType.argsTemplate) {
-      rhs = lhsType.argsTemplate(l, ...rhs);
+      rhs = lhsType.argsTemplate.bind(this.getState())(l, ...rhs);
     } else {
       rhs = `(${rhs.join(', ')})`;
     }
@@ -494,7 +513,7 @@ module.exports = (ANTLRVisitor) => class CodeGenerationVisitor extends ANTLRVisi
       ? lhsType.template()
       : defaultT;
     const rhs = lhsType.argsTemplate
-      ? lhsType.argsTemplate(lhsArg, ...args)
+      ? lhsType.argsTemplate.bind(this.getState())(lhsArg, ...args)
       : defaultA;
     const lhs = skipLhs ? '' : lhsArg;
     return this.Syntax.new.template
@@ -516,7 +535,7 @@ module.exports = (ANTLRVisitor) => class CodeGenerationVisitor extends ANTLRVisi
     let args = '';
     const keysAndValues = this.getKeyValueList(ctx);
     if (ctx.type.argsTemplate) {
-      args = ctx.type.argsTemplate(
+      args = ctx.type.argsTemplate.bind(this.getState())(
         this.getKeyValueList(ctx).map((k) => {
           return [this.getKeyStr(k), this.visit(this.getValue(k))];
         }),
@@ -526,7 +545,7 @@ module.exports = (ANTLRVisitor) => class CodeGenerationVisitor extends ANTLRVisi
     }
     ctx.indentDepth--;
     if (ctx.type.template) {
-      return ctx.type.template(args, ctx.indentDepth);
+      return ctx.type.template.bind(this.getState())(args, ctx.indentDepth);
     }
     return this.visitChildren(ctx);
   }
@@ -545,7 +564,7 @@ module.exports = (ANTLRVisitor) => class CodeGenerationVisitor extends ANTLRVisi
     if (ctx.type.argsTemplate) { // NOTE: not currently being used anywhere.
       args = visitedElements.map((arg, index) => {
         const last = !visitedElements[index + 1];
-        return ctx.type.argsTemplate(arg, ctx.indentDepth, last);
+        return ctx.type.argsTemplate.bind(this.getState())(arg, ctx.indentDepth, last);
       }).join('');
     } else {
       args = visitedElements.join(', ');

packages/bson-transpilers/codegeneration/DeclarationStore.js

@@ -0,0 +1,90 @@
+/**
+ * Stores declarations for use in the DriverTemplate
+ *
+ * @returns {object}
+ */
+class DeclarationStore {
+  constructor() {
+    this.clear();
+  }
+
+  /**
+   * Add declarations by templateID + varRoot + declaration combo.  Duplications will not be collected, rather the add
+   * method will return the existing declaration's variable name.
+   *
+   * @param {string} templateID - Name/alias of the template the declaration is being made for
+   * @param {string} varRoot - The root of the variable name to be appended by the occurrence count
+   * @param {function} declaration - The code block to be prepended to the driver syntax
+   * @returns {string} the variable name with root and appended count
+   */
+  addVar(templateID, varRoot, declaration) {
+    // Don't push existing declarations
+    const current = this.alreadyDeclared(templateID, varRoot, declaration);
+    if (current !== undefined) {
+      return current;
+    }
+    const varName = this.next(templateID, varRoot);
+    this.vars[declaration(varName)] = varName;
+    return varName;
+  }
+
+  /**
+   * Add a function to the funcs set
+   *
+   * @param {string} fn - String literal of a function
+   */
+  addFunc(fn) {
+    if (!this.funcs[fn]) {
+      this.funcs[fn] = true;
+    }
+  }
+
+  alreadyDeclared(templateID, varRoot, declaration) {
+    const existing = this.candidates(templateID, varRoot);
+    for (var i = 0; i < existing.length; i++) {
+      const candidate = `${this.varTemplateRoot(templateID, varRoot)}${i > 0 ? i : ''}`;
+      const current = this.vars[declaration(candidate)];
+      if (current !== undefined) {
+        return current;
+      }
+    }
+  }
+
+  candidates(templateID, varRoot) {
+    const varTemplateRoot = this.varTemplateRoot(templateID, varRoot);
+    return Object.values(this.vars).filter(varName => varName.startsWith(varTemplateRoot));
+  }
+
+  clear() {
+    this.vars = {};
+    this.funcs = {};
+  }
+
+  length() {
+    return Object.keys(this.vars).length + Object.keys(this.funcs).length;
+  }
+
+  next(templateID, varRoot) {
+    const existing = this.candidates(templateID, varRoot);
+
+    // If the data does not exist in the vars, then the count should append nothing to the variable name
+    const count = existing.length > 0 ? existing.length : '';
+    return `${this.varTemplateRoot(templateID, varRoot)}${count}`;
+  }
+
+  /**
+   * Stringify the variable declarations
+   *
+   * @param {string} sep - Seperator string placed between elements in the resulting string of declarations
+   * @returns {string} all the declarations as a string seperated by a line-break
+   */
+  toString(sep = '\n\n') {
+    return [...Object.keys(this.vars), ...Object.keys(this.funcs)].join(sep);
+  }
+
+  varTemplateRoot(templateID, varRoot) {
+    return `${varRoot}${templateID}`;
+  }
+}
+
+module.exports = DeclarationStore;

packages/bson-transpilers/codegeneration/object/Generator.js

@@ -92,7 +92,7 @@ module.exports = (Visitor) => class Generator extends Visitor {
     }
 
     if (lhsType && lhsType.argsTemplate) {
-      return lhsType.argsTemplate(lhs, ...args);
+      return lhsType.argsTemplate.bind(this.getState())(lhs, ...args);
     }
 
     let expr;
@@ -163,7 +163,7 @@ module.exports = (Visitor) => class Generator extends Visitor {
         return this.Syntax.equality.template(s, op, this.visit(arr[i + 1]));
       }
       if (op === 'in' || op === 'notin') {
-        return this.Syntax.in.template(s, op, this.visit(arr[i + 1]));
+        return this.Syntax.in.template.bind(this.state)(s, op, this.visit(arr[i + 1]));
       }
       throw new BsonTranspilersRuntimeError(`Unrecognized operation ${op}`);
     }, this.visit(ctx.children[0]));

packages/bson-transpilers/codegeneration/python/Visitor.js

@@ -171,7 +171,7 @@ module.exports = (CodeGenerationVisitor) => class Visitor extends CodeGeneration
       if (ctx.type.argsTemplate) { // NOTE: not currently being used anywhere.
         args = visitedElements.map((arg, index) => {
           const last = !visitedElements[index + 1];
-          return ctx.type.argsTemplate(arg, ctx.indentDepth, last);
+          return ctx.type.argsTemplate.bind(this.getState())(arg, ctx.indentDepth, last);
         });
         join = '';
       } else {
@@ -338,7 +338,7 @@ module.exports = (CodeGenerationVisitor) => class Visitor extends CodeGeneration
       if (op === 'in' || op === 'notin') {
         skip = true;
         if (this.Syntax.in) {
-          return `${str}${this.Syntax.in.template(
+          return `${str}${this.Syntax.in.template.bind(this.state)(
             this.visit(arr[i - 1]), op, this.visit(arr[i + 1]))}`;
         }
         return `${str} ${this.visit(arr[i - 1])} ${op} ${this.visit(arr[i + 1])}`;

packages/bson-transpilers/index.js

@@ -133,8 +133,9 @@ const getTranspiler = (loadTree, visitor, generator, symbols) => {
         idiomatic;
       if (!driverSyntax) {
         transpiler.clearImports();
+        transpiler.clearDeclarations();
       }
-      return transpiler.start(tree);
+      return transpiler.start(tree, !driverSyntax);
     } catch (e) {
       if (e.code && e.code.includes('BSONTRANSPILERS')) {
         throw e;
@@ -148,6 +149,7 @@ const getTranspiler = (loadTree, visitor, generator, symbols) => {
   return {
     compileWithDriver: (input, idiomatic) => {
       transpiler.clearImports();
+      transpiler.clearDeclarations();
 
       const result = {};
       Object.keys(input).map((k) => {
@@ -171,7 +173,7 @@ const getTranspiler = (loadTree, visitor, generator, symbols) => {
           'Generating driver syntax not implemented for current language'
         );
       }
-      return transpiler.Syntax.driver(result);
+      return transpiler.Syntax.driver.bind(transpiler.getState())(result);
     },
     compile: compile,
     getImports: (driverSyntax) => {