Imports with generation (#53)

* Imports resolving has been completely reworked; Generation for scripts with imports is now supported * Minor fix * Fixed imports' recursion
epam · Apr 23, 2019 · 1ea4afd · 1ea4afd
1 parent 6bca025
commit 1ea4afd
Show file tree

Hide file tree

Showing 18 changed files with 1,618 additions and 2,059 deletions.
diff --git a/src/generator/WDL/entities/ImportGenerator.js b/src/generator/WDL/entities/ImportGenerator.js
@@ -0,0 +1,31 @@
+import Settings from '../WDLSettings';
+
+export default class ImportGenerator {
+
+  constructor(imp, settings) {
+    this.name = imp.name;
+    this.expression = imp.expression;
+    this.uri = imp.uri;
+
+    this.importString = '';
+
+    this.settings = new Settings(settings);
+  }
+
+  renderImport() {
+    const EOL = this.settings.getValue('style.eol');
+
+    if (this.importString === '' && this.expression) {
+      this.importString += `${this.expression}${EOL}`;
+    }
+
+    return this.importString;
+  }
+
+  renderEOL() {
+    const EOL = this.settings.getValue('style.eol');
+
+    return `${EOL}`;
+  }
+
+}
diff --git a/src/generator/WDL/entities/WorkflowGenerator.js b/src/generator/WDL/entities/WorkflowGenerator.js
@@ -13,7 +13,7 @@ export default class WorkflowGenerator {
       scatter: this.genScatter,
       whileloop: this.genWhile,
       if: this.genIf,
-      workflow: this.genWorkflow,
+      workflow: this.genCall,
     };
     this.wfName = wfStep.name;
     this.wfStep = wfStep;
@@ -103,9 +103,7 @@ export default class WorkflowGenerator {
   }
 
   buildPortValue(value) {
-    if (value.desc && value.desc.default && !_.isUndefined(value.desc.default)) {
-      return `${value.desc.default}`;
-    } else if (value.desc && value.desc.expression && !_.isUndefined(value.desc.expression)) {
+    if (value.desc && value.desc.expression && !_.isUndefined(value.desc.expression)) {
       return `${value.desc.expression}`;
     } else if (value.expression && value.expression.type.toLowerCase() !== 'identifier'
       && value.expression.type.toLowerCase() !== 'memberaccess' && !_.isUndefined(value.expression.string)) {
@@ -121,7 +119,7 @@ export default class WorkflowGenerator {
         const outStepName = outStep.name;
         const outVarName = connection.from.name;
 
-        if (outStepName === this.wfName || outStep.type) {
+        if (outStepName === this.wfName || (outStep.type && outStep.type.toLowerCase() !== constants.WORKFLOW)) {
           return outVarName;
         }
         return `${outStepName}.${outVarName}`;
@@ -140,12 +138,17 @@ export default class WorkflowGenerator {
     let res = '';
 
     const val = child;
+    const namespace = val.namespace ? `${val.namespace}.` : '';
     let callString = `${SCOPE_INDENT}${constants.CALL} `;
 
-    if (val.action.name === val.name) {
-      callString += `${val.name}`;
+    if (val.name === val.initialName || val.initialName === `${namespace}${val.name}`) {
+      if (val.action.name === `${namespace}${val.name}`) {
+        callString += `${namespace}${val.name}`;
+      } else {
+        callString += `${val.action.name} ${constants.AS} ${val.name}`;
+      }
     } else {
-      callString += `${val.action.name} ${constants.AS} ${val.name}`;
+      callString += `${val.initialName} ${constants.AS} ${val.name}`;
     }
 
     res += `${callString}`;
@@ -234,11 +237,6 @@ export default class WorkflowGenerator {
     return res;
   }
 
-  genWorkflow(child) {
-    // TODO: add generation workflow
-    this.genCall(child);
-  }
-
   buildOutputMap(outputMappings) {
     const EOL = this.settings.getValue('style.eol');
     const SCOPE_INDENT = this.settings.getValue('style.scope_indent');

diff --git a/src/generator/WDL/generate.js b/src/generator/WDL/generate.js
@@ -2,22 +2,34 @@ import _ from 'lodash';
 
 import TaskGenerator from './entities/TaskGenerator';
 import WorkflowGenerator from './entities/WorkflowGenerator';
+import ImportGenerator from './entities/ImportGenerator';
 
 export default function generate(objectModel) {
   const settings = {};
 
+  let imports = '';
+  _.forEach(objectModel.imports, (imp, i) => {
+    const importGen = new ImportGenerator(imp, settings);
+    imports += importGen.renderImport();
+    if (i === objectModel.imports.length - 1) {
+      imports += importGen.renderEOL();
+    }
+  });
+
   const wf = new WorkflowGenerator(objectModel, settings).renderWorkflow();
 
   let tasks = '';
   const actionsToBeRendered = {};
 
   const actionSelector = (children) => {
     _.forEach(children, (val) => {
-      if (!val.type) {
-        const action = val.action;
-        actionsToBeRendered[action.name] = action;
-      } else {
-        actionSelector(val.children);
+      if (!val.imported) {
+        if (!val.type) {
+          const action = val.action;
+          actionsToBeRendered[action.name] = action;
+        } else {
+          actionSelector(val.children);
+        }
       }
     });
   };
@@ -28,5 +40,5 @@ export default function generate(objectModel) {
     tasks += new TaskGenerator(action).renderTask();
   });
 
-  return wf + tasks;
+  return imports + wf + tasks;
 }
diff --git a/src/generator/generate.js b/src/generator/generate.js
@@ -10,9 +10,6 @@ import generateWDL from './WDL/generate';
  * @returns {string} Textual representation of the workflow.
  */
 function generate(flow, opts = {}) {
-  if (flow.hasImports) {
-    throw new Error('Generation for scripts containing imports is not supported');
-  }
   const format = opts.format || 'wdl';
   if (format === 'wdl') {
     return generateWDL(flow, opts);

diff --git a/src/model/Action.js b/src/model/Action.js
@@ -20,6 +20,7 @@ function createPort(portDesc) {
   return _.defaults(_.pick(portDesc, [
     'type',
     'default',
+    'expression',
     'multi',
   ]), {
     multi: false,

diff --git a/src/model/Step.js b/src/model/Step.js
@@ -1,6 +1,7 @@
 import _ from 'lodash';
 import Action from './Action';
 import Port from './Port';
+import { extractNamespace, extractName } from '../parser/WDL/utils/utils';
 
 /**
  * Bind ports during step construction.
@@ -100,8 +101,9 @@ export default class Step {
    * @param {object} config - Action configuration containing input bindings.
    * It should include action description in case the action is missing.
    *
+   * @param initialName
    */
-  constructor(name, action = {}, config = {}) {
+  constructor(name, action = {}, config = {}, initialName = '') {
     if (_.isUndefined(name)) {
       throw new Error('Step must have a name');
     }
@@ -113,11 +115,29 @@ export default class Step {
     action.on('changed', () => this._onActionChanged());
     action.on('port-rename', (...args) => this._onPortRename(...args));
 
+    /**
+     * Step namespace.
+     * @type {string}
+     */
+    this.namespace = extractNamespace(initialName) || extractNamespace(name);
+
+    /**
+     * Step initial name.
+     * @type {string}
+     */
+    this.initialName = initialName || action.name;
+
+    /**
+     * Shows if Step was imported.
+     * @type {Boolean}
+     */
+    this.imported = this.initialName !== extractName(this.initialName);
+
     /**
      * Step name.
      * @type {string}
      */
-    this.name = name;
+    this.name = extractName(name);
     /**
      * Action performed during the step.
      * @type {Action}

diff --git a/src/model/Workflow.js b/src/model/Workflow.js
@@ -1,5 +1,6 @@
 import _ from 'lodash';
 import Group from './Group';
+import { extractNamespace } from '../parser/WDL/utils/utils';
 
 /**
  * Class representing the entire workflow.
@@ -26,7 +27,7 @@ class Workflow extends Group {
   constructor(name, config = {}) {
     super(name, 'workflow', config);
     /**
-     * A dictionary of actions to be (probably) used in the workflow.
+     * A dictionary of actions to be used in the workflow.
      *
      * It is updated automatically when you {@link Workflow#add add} a child or a grandchild step.
      * You may also wish to {@link Workflow#addAction add actions} manually even if they do not correspond
@@ -46,6 +47,7 @@ class Workflow extends Group {
     }
     if (config.initialName) {
       this.initialName = config.initialName;
+      this.namespace = extractNamespace(config.initialName);
     }
 
     this.isSubWorkflow = config.isSubWorkflow || false;

diff --git a/src/parser/WDL/entities/Context.js b/src/parser/WDL/entities/Context.js
@@ -2,7 +2,7 @@ import _ from 'lodash';
 
 import WDLWorkflow from './WDLWorkflow';
 import Task from './Task';
-import { WDLParserError, extractExpression, extractType } from '../utils/utils';
+import { WDLParserError, extractExpression, extractType, extractImportsArray } from '../utils/utils';
 import Workflow from '../../../model/Workflow';
 import * as Constants from '../constants';
 
@@ -12,10 +12,10 @@ export default class Context {
    * Process through the hole entire ast tree and builds the desired Object Model
    * @param {ast} ast - Root ast tree node of parsing result
    * @param {string} src - Source WDL string (needed to extract the task command strings)
+   * @param {Object} imports - Object of parsed imports
    */
-  constructor(ast, src) {
-    this.genericTaskCommandMap = new Map();
-    this.preprocessTheTaskOntoCommandMap(src);
+  constructor(ast, src, imports = {}) {
+    this.imports = Context.preprocessCommandMapsRecursively(imports, src);
     this.actionMap = this.buildActionMap(ast);
 
     try {
@@ -25,14 +25,24 @@ export default class Context {
     }
   }
 
+  static preprocessCommandMapsRecursively(importsDict, src) {
+    const imports = importsDict;
+    imports.genericTaskCommandMap = Context.preprocessTheTaskOntoCommandMap(src);
+    if (imports.children) {
+      Object.keys(imports.children).forEach((childName) => {
+        imports.children[childName] =
+          Context.preprocessCommandMapsRecursively(imports.children[childName], imports.children[childName].src);
+      });
+    }
+
+    return imports;
+  }
+
   /**
    * Process through the workflow entities
    * @param {ast} ast - Root ast tree node of parsing result
    */
   buildWorkflowList(ast) {
-    if (ast.attributes.imports && ast.attributes.imports.list.length) {
-      this.hasImports = true;
-    }
     const definitions = ast.attributes.body;
     return definitions ? definitions.list
       .filter(item => item.name.toLowerCase() === 'workflow')
@@ -44,8 +54,8 @@ export default class Context {
    *    (TBD: Not safe. Not all cases supported)
    * @param {string} src - Source WDL script string
    */
-  preprocessTheTaskOntoCommandMap(src) {
-    this.genericTaskCommandMap = new Map();
+  static preprocessTheTaskOntoCommandMap(src) {
+    const genericTaskCommandMap = new Map();
 
     const taskRegex = /\s*^task\s*.+\s*\{/gm;
     const tasks = [];
@@ -81,20 +91,22 @@ export default class Context {
 
       if (currCmd &&
           (!nextTask || (val.lastIndex < currCmd.lastIndex && currCmd.lastIndex < nextTask.lastIndex))) {
-        this.genericTaskCommandMap.set(val.message, currCmd);
+        genericTaskCommandMap.set(val.message, currCmd);
         cmdIdx += 1;
       }
     });
 
-    this.genericTaskCommandMap.forEach((val, key) => {
+    genericTaskCommandMap.forEach((val, key) => {
       const lastIndex = val.lastIndex;
       const closer = val.type === '{' ? '}' : '>>>';
 
-      this.genericTaskCommandMap.set(key, {
+      genericTaskCommandMap.set(key, {
         command: Context.traceTheCommand(src, lastIndex, closer),
         type: val.type,
       });
     });
+
+    return genericTaskCommandMap;
   }
 
   static traceTheCommand(str, start, closer) {
@@ -125,9 +137,13 @@ export default class Context {
   /**
    * Convert WDL tasks to Actions and store them in map for future using
    * @param {ast} ast - Root ast tree node of parsing result
+   * @param {Object} imports
    */
-  buildActionMap(ast) {
+  buildActionMap(ast, imports = this.imports) {
     const actionMap = {};
+    if (!ast) {
+      return actionMap;
+    }
     const definitions = ast.attributes.body;
     const tasks = definitions.list.filter(item => item.name.toLowerCase() === Constants.TASK)
       .map(wfNode => new Task(wfNode.attributes));
@@ -141,17 +157,41 @@ export default class Context {
       });
 
     tasks.forEach((task) => {
-      const command = this.genericTaskCommandMap.get(task.name);
+      const command = imports.genericTaskCommandMap.get(task.name);
       actionMap[task.name] = task.constructAction(command);
     });
 
     workflows.forEach((workflow) => {
       actionMap[workflow.name] = workflow;
     });
 
+    const importsArray = extractImportsArray(ast);
+    if (importsArray.length) {
+      importsArray.forEach((imp) => {
+        const childImport = Context.findChildImport(imp.name, imports);
+        if (childImport) {
+          const childActionMap = this.buildActionMap(childImport.ast, childImport);
+          _.forEach(childActionMap, (action, name) => {
+            const actionName = `${imp.name}.${name}`;
+            action.name = actionName;
+            actionMap[actionName] = action;
+          });
+        }
+      });
+    }
+
     return actionMap;
   }
 
+  static findChildImport(namespace, imports) {
+    if (imports && imports.imports.length) {
+      if (imports.children[namespace]) {
+        return imports.children[namespace];
+      }
+    }
+    return null;
+  }
+
   /**
    * Get all workflow inputs
    * @param {ast} wfNodeBody - ast tree node.attributes.body