Merge 99cf425 into ed0672a

tools/python/boutiques/bosh.py

@@ -56,6 +56,9 @@ def parser_create():
     parser.add_argument("--use-singularity", '-u', action="store_true",
                         help="When --docker-image is used. Specify to "
                              "use singularity to run it.")
+    parser.add_argument("--cl-template", action="store",
+                        help="Path to descriptor from which its command-line"
+                        " template is used to generate descriptor inputs.")
     parser.add_argument("--camel-case", action="store_true",
                         help="All input IDs will be written in camelCase.")
     return parser
@@ -69,7 +72,8 @@ def create(*params):
     new = CreateDescriptor(parser=None,
                            docker_image=results.docker_image,
                            use_singularity=results.use_singularity,
-                           camel_case=results.camel_case)
+                           camel_case=results.camel_case,
+                           cl_template=results.cl_template)
     new.save(results.descriptor)
     return None
 
@@ -342,16 +346,18 @@ def parser_importer():
                             " CWL descriptor or docopt script to spec.")
     parser.add_argument("type", help="Type of import we are performing."
                         " Allowed values: {" +
-                        ", ".join(["bids", "0.4", "cwl", "docopt"]) + "}",
-                        choices=["bids", "0.4", "cwl", "docopt"],
+                        ", ".join(["bids", "0.4", "cwl", "docopt", "config"]) +
+                        "}",
+                        choices=["bids", "0.4", "cwl", "docopt", "config"],
                         metavar='type')
     parser.add_argument("output_descriptor", help="Where the Boutiques"
                         " descriptor will be written.")
     parser.add_argument("input_descriptor", help="Input descriptor to be"
                         " converted. For '0.4', is JSON descriptor,"
-                        " for 'docopt' is JSON descriptor,"
+                        " for 'docopt' is Docopt script,"
                         " for 'bids' is base directory of BIDS app,"
-                        " for 'cwl' is YAML descriptor.")
+                        " for 'cwl' is YAML descriptor,"
+                        " for 'config' is configuration file {.json, .toml, .yml}.")
     parser.add_argument("-o", "--output-invocation", help="Where to write "
                         "the invocation if any.")
     parser.add_argument("-i", "--input-invocation", help="Input invocation "
@@ -375,8 +381,11 @@ def importer(*params):
     elif results.type == "cwl":
         importer.import_cwl()
     elif results.type == "docopt":
-        create(params[1])
-        importer.import_docopt(params[1])
+        create(importer.output_descriptor)
+        importer.import_docopt()
+    elif results.type == "config":
+        create(importer.output_descriptor)
+        importer.import_config()
 
 
 importer.__doc__ = parser_importer().format_help()

tools/python/boutiques/creator.py

@@ -4,12 +4,16 @@
 import argparse
 import sys
 import os
+import re
+import yaml
+import toml
 import simplejson as json
 import os.path as op
 from jsonschema import validate, ValidationError
 from argparse import ArgumentParser
 from boutiques import __file__ as bfile
 from boutiques.logger import raise_error, print_info, print_warning
+from boutiques.util.utils import loadJson
 from boutiques.util.utils import customSortDescriptorByKey, camelCaseInputIds
 import subprocess
 
@@ -47,6 +51,9 @@ def __init__(self, parser=None, **kwargs):
         if self.camelCase:
             self.descriptor = camelCaseInputIds(self.descriptor)
 
+        if kwargs.get('cl_template'):
+            self.generateInputsFromTemplate(kwargs.get('cl_template'))
+
     def save(self, filename):
         with open(filename, "w") as f:
             f.write(json.dumps(
@@ -270,3 +277,40 @@ def executor(self, command):
             raise e
         else:
             return process.communicate(), process.returncode
+
+    def generateInputsFromTemplate(self, cl_template_path):
+        def _createIdFromValueKey(vk):
+            vk = re.sub(r"\[(\w+)\]", r"\1", vk).lower()
+            vk_words = vk.split("_")
+            return vk_words[0] + ''.join(x.title() for x in vk_words[1:])
+
+        def _createNameFromId(id):
+            name = [c for c in id]
+            for idx, c in enumerate(name):
+                if not c.isalnum():
+                    name[idx] = ' '
+                if c.isupper():
+                    name[idx] = ' ' + c.lower()
+            # Join char list into string and concatenate white spaces
+            return " ".join(''.join(name).split()).title()
+
+        def _getValueKeys(config_file):
+            # Return all value-key occurences, determined by brackets
+            template = "".join(config_file['file-template'])
+            return re.findall(r"\[\w+\]", template)
+
+        template_descriptor = loadJson(cl_template_path)
+        # Get file-template from first output file with 'file-template'
+        config_file = next(out for out in template_descriptor['output-files']
+                           if 'file-template' in out)
+
+        # Clear descriptor inputs and repopulate
+        self.descriptor['inputs'] = []
+        for vk in _getValueKeys(config_file):
+            generated_id = _createIdFromValueKey(vk)
+            self.descriptor['inputs'].append({
+                'id': generated_id,
+                'name': _createNameFromId(generated_id),
+                'type': "String",
+                'value-key': vk
+            })

tools/python/boutiques/importer.py

@@ -8,6 +8,7 @@
 from boutiques.logger import raise_error
 import boutiques
 import yaml
+import toml
 import simplejson as json
 import os
 import os.path as op
@@ -125,8 +126,8 @@ def get_entry_point(self, input_descriptor):
 
         return entrypoint
 
-    def import_docopt(self, desc_path):
-        template_file = op.join(desc_path)
+    def import_docopt(self):
+        template_file = op.join(self.output_descriptor)
         docstring = imp.load_source(
             'docopt_pyscript', self.input_descriptor).__doc__
 
@@ -449,6 +450,147 @@ def get_input(descriptor_inputs, input_id):
         boutiques.invocation(self.output_descriptor,
                              "-i", self.output_invocation)
 
+    def import_config(self):
+        def _getConfigFileString():
+            with open(self.input_descriptor, "r") as configFile:
+                return configFile.read()
+
+        def _getPropertiesFromValue(value):
+            # Generate input properties from the input's value
+            if isinstance(value, bool):
+                return {'type': "Flag",
+                        "command-line-flag": "--TEMP",
+                        "optional": True}
+            if isinstance(value, float):
+                return {'type': "Number", 'integer': False}
+            elif isinstance(value, int):
+                return {'type': "Number", 'integer': True}
+            elif isinstance(value, str):
+                if re.match(r"\/.*\.[\w:]+", value) is not None:
+                    return {'type': "File"}
+                return {'type': "String"}
+            elif isinstance(value, list):
+                # Check all elements of list to extract list properties
+                elementProperties, props = {}, []
+                for element in value:
+                    props.append(_getPropertiesFromValue(element))
+                if all([p['type'] == "Flag" for p in props]):
+                    elementProperties['type'] = "Flag"
+                    elementProperties['command-line-flag'] = "--TEMP"
+                    elementProperties['optional'] = True
+                elif all([p['type'] == "Number" for p in props]):
+                    elementProperties['type'] = "Number"
+                    elementProperties['integer'] =\
+                        all([p['integer'] for p in props])
+                elif all([p['type'] == "File" for p in props]):
+                    elementProperties['type'] = "File"
+                else:
+                    elementProperties['type'] = "String"
+                elementProperties['list'] = True
+                return elementProperties
+
+        def _createNameFromID(id):
+            name = [c for c in id]
+            for idx, c in enumerate(name):
+                if not c.isalnum():
+                    name[idx] = ' '
+                if c.isupper():
+                    name[idx] = ' ' + c.lower()
+            # Join char list into string and concatenate white spaces
+            return " ".join(''.join(name).split()).title()
+
+        def _getOutputConfigFileTemplate(configFileFormat):
+            return {
+                "id": "config_file",
+                "name": "Configuration file",
+                "value-key": "[CONFIG_FILE]",
+                "path-template": "config.{0}".format(configFileFormat),
+                "file-template": []
+            }
+
+        def _getInputsFromConfigDict(input_config):
+            # Generate inputs based on input config file (as dict)
+            desc_inputs = []
+            for id, value in input_config.items():
+                newInput = {'id': id, 'name': _createNameFromID(id)}
+                newInput.update(_getPropertiesFromValue(value))
+                newInput['value-key'] =\
+                    "[{0}]".format(newInput['name'].replace(' ', '_').upper())
+                desc_inputs.append(newInput)
+            return desc_inputs
+
+        def import_json(descriptor):
+            input_config = loadJson(self.input_descriptor)
+            imported_inputs = _getInputsFromConfigDict(input_config)
+            descriptor['inputs'].extend(imported_inputs)
+
+            # file-template formatting depends on the config file's format
+            output_config_file = _getOutputConfigFileTemplate(configFileFormat)
+            output_config_file['file-template'].append('{')
+            for inp in descriptor['inputs']:
+                input_entry = "\"{0}\": \"{1}\"".format(inp['id'], inp['value-key'])
+                if inp != descriptor['inputs'][-1]:
+                    input_entry += ","
+                output_config_file['file-template'].append(input_entry)
+            output_config_file['file-template'].append('}')
+
+            descriptor['output-files'].append(output_config_file)
+            return descriptor
+
+        def import_toml(descriptor):
+            tomlString = _getConfigFileString()
+            input_config = toml.loads(tomlString)
+            imported_inputs = _getInputsFromConfigDict(input_config)
+            descriptor['inputs'].extend(imported_inputs)
+
+            # file-template formatting depends on the config file's format
+            output_config_file = _getOutputConfigFileTemplate(configFileFormat)
+            for inp in descriptor['inputs']:
+                output_config_file['file-template'].append(
+                    "\"{0}\"=\"{1}\"".format(inp['id'], inp['value-key']))
+
+            descriptor['output-files'].append(output_config_file)
+            return descriptor
+
+        def import_yaml(descriptor):
+            yamlString = _getConfigFileString()
+            input_config = yaml.load(yamlString, Loader=yaml.FullLoader)
+            imported_inputs = _getInputsFromConfigDict(input_config)
+            descriptor['inputs'].extend(imported_inputs)
+
+            # file-template formatting depends on the config file's format
+            output_config_file = _getOutputConfigFileTemplate(configFileFormat)
+            for inp in descriptor['inputs']:
+                output_config_file['file-template'].append(
+                    "\"{0}\": {1}".format(inp['id'], inp['value-key']))
+
+            descriptor['output-files'].append(output_config_file)
+            return descriptor
+
+        descriptor = loadJson(self.output_descriptor)
+        # Remove inputs and output-files from descriptor
+        # Not emptying inputs also works, can be removed if needed
+        descriptor['inputs'], descriptor['output-files'] = [], []
+        configFileFormat = self.input_descriptor.split(".")[-1].lower()
+
+        # Config K:V pair loading changes depending on config file type
+        # Populating descriptor with inputs and configuration file
+        if configFileFormat == "json":
+            descriptor = import_json(descriptor)
+        elif configFileFormat == "toml":
+            descriptor = import_toml(descriptor)
+        elif configFileFormat == "yml":
+            descriptor = import_yaml(descriptor)
+
+        descriptor['command-line'] = "tool "
+        descriptor['command-line'] += " ".join(
+            ["{0}".format(inp['value-key']) for inp in descriptor['inputs']])
+        descriptor['command-line'] += " [CONFIG_FILE]"
+
+        with open(self.output_descriptor, "w+") as output:
+            output.write(json.dumps(
+                customSortDescriptorByKey(descriptor), indent=4))
+
 
 class Docopt_Importer():
     def __init__(self, docopt_str, base_descriptor):

tools/python/boutiques/tests/config/configuration.json

@@ -0,0 +1,10 @@
+{
+    "numInput": 5050,
+    "flagInput": true,
+    "listNumInput": [1,2,3,4,5],
+    "stringInput": "foo",
+    "fileInput": "/somewhere/something.txt",
+    "optStringInput": "Hello, world!",
+    "fileOutput": "/somewhere/somethingelse.txt",
+    "doubleInput": 50.5
+}

tools/python/boutiques/tests/config/configuration.toml

@@ -0,0 +1,13 @@
+# This is a demo configuration file
+numInput=5050
+flagInput=true
+# This is a comment
+listNumInput=[1,2,3,4,5]
+# This input is hard-coded
+stringInput="foo"
+fileInput="/somewhere/something.txt"
+# This is an optional input. Line will be removed when value is not defined.
+optStringInput="Hello, world!"
+# And here is the result
+fileOutput="/somewhere/somethingelse.txt"
+doubleInput=50.5

tools/python/boutiques/tests/config/configuration.yml

@@ -0,0 +1,13 @@
+# This is a demo configuration file
+numInput: 5050
+flagInput: true
+# This is a comment
+listNumInput: [1,2,3,4,5]
+# This input is hard-coded
+stringInput: "foo"
+fileInput: "/somewhere/something.txt"
+# This is an optional input. Line will be removed when value is not defined.
+optStringInput: "Hello, world!"
+# And here is the result
+fileOutput: "/somewhere/somethingelse.txt"
+doubleInput: 50.5