Skip to content

Commit

Permalink
Merge pull request #623 from DarrinFong/refactoring_bosh_parsers
Browse files Browse the repository at this point in the history 
Refactoring bosh parsers (2)
  • Loading branch information
@glatard
glatard committed Jul 2, 2020
2 parents 71078de + e27d5a1 commit 4d4bb27
Show file tree
Hide file tree
Showing 18 changed files with 898 additions and 805 deletions.
3 changes: 2 additions & 1 deletion .gitignore
View file
Expand Up @@ -13,4 +13,5 @@ dist/
*~
temp-*.localExec.boshjob.sh
.vscode
.idea/
.idea/
boutiques-example1-test.simg
757 changes: 138 additions & 619 deletions tools/python/boutiques/bosh.py
View file

Large diffs are not rendered by default.

613 changes: 613 additions & 0 deletions tools/python/boutiques/boshParsers.py
View file

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions tools/python/boutiques/dataHandler.py
View file
Expand Up @@ -296,3 +296,7 @@ def getDataCacheDir():
if not os.path.exists(data_cache_dir):
os.makedirs(data_cache_dir, exist_ok=True)
return data_cache_dir


class DataHandlerError(Exception):
pass
35 changes: 18 additions & 17 deletions tools/python/boutiques/importer.py
View file
Expand Up @@ -17,7 +17,7 @@
from docopt import parse_defaults, parse_pattern, parse_argv
from docopt import formal_usage, DocoptLanguageError
from docopt import AnyOptions, TokenStream, Option, Argument, Command
import imp
from importlib.machinery import SourceFileLoader
import collections


Expand Down Expand Up @@ -95,7 +95,7 @@ def upgrade_04(self):
with open(self.output_descriptor, 'w') as fhandle:
fhandle.write(json.dumps(
customSortDescriptorByKey(descriptor), indent=4))
validate_descriptor(self.output_descriptor)
validate_descriptor(loadJson(self.output_descriptor))

def get_entry_point(self, input_descriptor):
entrypoint = None
Expand Down Expand Up @@ -127,18 +127,11 @@ def get_entry_point(self, input_descriptor):
return entrypoint

def import_docopt(self):
template_file = op.join(self.output_descriptor)
docstring = imp.load_source(
'docopt_pyscript', self.input_descriptor).__doc__

docoptImporter = Docopt_Importer(docstring, template_file)
# The order matters
docoptImporter.loadDocoptDescription()
docoptImporter.loadDescriptionAndType()
docoptImporter.generateInputsAndCommandLine(docoptImporter.pattern)
docoptImporter.addInputsRecursive(docoptImporter.dependencies)
docoptImporter.determineOptionality()
docoptImporter.createRootOneIsRequiredGroup()
# input_descriptor is the .py containing docopt script
docstring = SourceFileLoader(
'docopt_script', self.input_descriptor).load_module().__doc__
# Init docopt importer and start import
docoptImporter = Docopt_Importer(docstring, self.output_descriptor)

with open(self.output_descriptor, "w") as output:
output.write(json.dumps(docoptImporter.descriptor, indent=4))
Expand Down Expand Up @@ -186,7 +179,8 @@ def import_cwl(self):

# Read the CWL descriptor
with open(self.input_descriptor, 'r') as f:
cwl_desc = json.loads(json.dumps(yaml.load(f)))
cwl_desc = json.loads(
json.dumps(yaml.load(f, Loader=yaml.FullLoader)))

# validate yaml descriptor?

Expand Down Expand Up @@ -424,7 +418,7 @@ def parse_req(req, req_type, bout_desc):
with open(self.output_descriptor, 'w') as f:
f.write(json.dumps(
customSortDescriptorByKey(bout_desc), indent=4))
validate_descriptor(self.output_descriptor)
validate_descriptor(loadJson(self.output_descriptor))

if self.input_invocation is None:
return
Expand All @@ -437,7 +431,7 @@ def get_input(descriptor_inputs, input_id):
return False
boutiques_invocation = {}
with open(self.input_invocation, 'r') as f:
cwl_inputs = json.loads(json.dumps(yaml.load(f)))
cwl_inputs = yaml.load(f, Loader=yaml.FullLoader)
for input_name in cwl_inputs:
if get_input(bout_desc['inputs'], input_name)['type'] != "File":
input_value = cwl_inputs[input_name]
Expand Down Expand Up @@ -630,6 +624,13 @@ def __init__(self, docopt_str, base_descriptor):
os.remove(base_descriptor)
raise_error(ImportError, "Invalid docopt script")

self.loadDocoptDescription()
self.loadDescriptionAndType()
self.generateInputsAndCommandLine(self.pattern)
self.addInputsRecursive(self.dependencies)
self.determineOptionality()
self.createRootOneIsRequiredGroup()

def loadDocoptDescription(self):
# Get description of the script itself
# which is everything except for usage and arguments
Expand Down
1 change: 0 additions & 1 deletion tools/python/boutiques/invocationSchemaHandler.py
View file
Expand Up @@ -10,7 +10,6 @@
import argparse
from functools import reduce
from jsonschema import ValidationError
from boutiques.validator import validate_descriptor
from boutiques.logger import raise_error, print_info


Expand Down
6 changes: 3 additions & 3 deletions tools/python/boutiques/publisher.py
View file
Expand Up @@ -3,7 +3,7 @@
from boutiques.validator import validate_descriptor, ValidationError
from boutiques.logger import raise_error, print_info
from boutiques.zenodoHelper import ZenodoError, ZenodoHelper
from boutiques.util.utils import customSortDescriptorByKey
from boutiques.util.utils import customSortDescriptorByKey, loadJson
import simplejson as json
import requests
import os
Expand All @@ -21,6 +21,7 @@ def __init__(self, descriptor_file_name, auth_token,
self.no_int = no_int
self.zenodo_access_token = auth_token
self.zenodo_helper = ZenodoHelper(sandbox, no_int, verbose)
self.descriptor = loadJson(self.descriptor_file_name)

# remove zenodo prefix of ID to update
try:
Expand All @@ -30,8 +31,7 @@ def __init__(self, descriptor_file_name, auth_token,
"'zenodo', e.g. zenodo.123456")

# Validate and load descriptor
validate_descriptor(descriptor_file_name)
self.descriptor = json.loads(open(self.descriptor_file_name).read())
validate_descriptor(self.descriptor)

# Get relevant descriptor properties
self.url = self.descriptor.get('url')
Expand Down
34 changes: 20 additions & 14 deletions tools/python/boutiques/tests/test_bosh.py
View file
Expand Up @@ -5,22 +5,28 @@
from unittest import TestCase
from boutiques import __file__ as bfile
from boutiques.bosh import bosh
from boutiques.localExec import ExecutorError
from boutiques.dataHandler import DataHandlerError
from boutiques import BoutiquesError


class TestBosh(TestCase):

def test_help(self):
self.assertRaises(SystemExit, bosh, [])
self.assertRaises(SystemExit, bosh, ["--help"])
self.assertRaises(SystemExit, bosh, ["exec", "--help"])
self.assertRaises(SystemExit, bosh, ["exec", "launch",
"--help"])
self.assertRaises(SystemExit, bosh, ["exec", "simulate",
"--help"])
self.assertRaises(SystemExit, bosh, ["publish", "--help"])
self.assertRaises(SystemExit, bosh, ["import", "--help"])
self.assertRaises(SystemExit, bosh, ["validate", "--help"])
self.assertRaises(SystemExit, bosh, ["invocation", "--help"])
self.assertRaises(SystemExit, bosh, ["evaluate", "--help"])
self.assertRaises(SystemExit, bosh, ["create", "--help"])
self.assertRaises(SystemExit, bosh, ["example", "--help"])
self.assertRaises(ExecutorError, bosh, [])
self.assertRaises(ExecutorError, bosh, ["--help"])
self.assertRaises(ExecutorError, bosh, ["exec", "--help"])
self.assertRaises(ExecutorError, bosh, ["exec", "launch", "--help"])
self.assertRaises(ExecutorError, bosh, ["exec", "simulate", "--help"])
self.assertRaises(ExecutorError, bosh, ["exec", "prepare", "--help"])
self.assertRaises(DataHandlerError, bosh, ["data", "--help"])
self.assertRaises(DataHandlerError, bosh, ["data", "delete", "--help"])
self.assertRaises(DataHandlerError, bosh, ["data", "inspect", "--help"])
self.assertRaises(DataHandlerError, bosh, ["data", "publish", "--help"])
self.assertRaises(BoutiquesError, bosh, ["publish", "--help"])
self.assertRaises(BoutiquesError, bosh, ["import", "--help"])
self.assertRaises(BoutiquesError, bosh, ["validate", "--help"])
self.assertRaises(BoutiquesError, bosh, ["invocation", "--help"])
self.assertRaises(BoutiquesError, bosh, ["evaluate", "--help"])
self.assertRaises(BoutiquesError, bosh, ["create", "--help"])
self.assertRaises(BoutiquesError, bosh, ["example", "--help"])
37 changes: 15 additions & 22 deletions tools/python/boutiques/tests/test_exec.py
View file
Expand Up @@ -4,6 +4,7 @@
from unittest import TestCase
from boutiques import __file__ as bfile
import boutiques as bosh
from boutiques.localExec import ExecutorError


class TestExec(TestCase):
Expand All @@ -14,25 +15,17 @@ def get_examples_dir(self):

def test_failing_launch(self):
example1_dir = os.path.join(self.get_examples_dir(), "example1")
self.assertRaises(SystemExit, bosh.execute, ["launch",
os.path.join(example1_dir,
"fake.json"),
os.path.join(
example1_dir,
"invocation.json"),
"--skip-data-collection"])
self.assertRaises(SystemExit, bosh.execute, ["launch",
os.path.join(
example1_dir,
"example1_docker.json"),
os.path.join(example1_dir,
"fake.json"),
"--skip-data-collection"])
self.assertRaises(SystemExit, bosh.execute, ["launch",
os.path.join(
example1_dir,
"example1_docker.json"),
os.path.join(
example1_dir,
"exampleTool1.py"),
"--skip-data-collection"])
self.assertRaises(ExecutorError, bosh.execute,
("launch", os.path.join(example1_dir, "fake.json"),
os.path.join(example1_dir, "invocation.json"),
"--skip-data-collection"))
self.assertRaises(ExecutorError, bosh.execute,
("launch", os.path.join(example1_dir,
"example1_docker.json"),
os.path.join(example1_dir, "fake.json"),
"--skip-data-collection"))
self.assertRaises(ExecutorError, bosh.execute,
("launch", os.path.join(example1_dir,
"example1_docker.json"),
os.path.join(example1_dir, "exampleTool1.py"),
"--skip-data-collection"))
6 changes: 5 additions & 1 deletion tools/python/boutiques/tests/test_publisher.py
View file
Expand Up @@ -190,9 +190,13 @@ def test_publication_replace_with_id(self, mock_get, mock_post, mock_put,

def test_publication_errors(self):
# Update an already published descriptor (wrong id)
example1_dir = op.join(self.get_examples_dir(), "example1")
example1_desc = op.join(example1_dir, "example1_docker.json")
temp_descriptor = tempfile.NamedTemporaryFile(suffix=".json")
shutil.copyfile(example1_desc, temp_descriptor.name)
with self.assertRaises(ZenodoError) as e:
wrong_id = bosh(["publish",
"whatever.json",
temp_descriptor.name,
"--sandbox", "-y", "-v",
"--zenodo-token", "hAaW2wSBZMskxpfigTYHcuDrC"
"PWr2VeQZgBLErKbfF5RdrKhzzJ"
Expand Down
88 changes: 37 additions & 51 deletions tools/python/boutiques/tests/test_simulate.py
View file
Expand Up @@ -8,6 +8,7 @@
from boutiques import __file__ as bfile
import boutiques as bosh
import mock
from boutiques.localExec import ExecutorError
from boutiques_mocks import mock_zenodo_search, MockZenodoRecord,\
example_boutiques_tool

Expand Down Expand Up @@ -56,7 +57,7 @@ def test_success_default_values(self):
"-i",
os.path.join(example1_dir,
"inv_no_defaults.json"))
.exit_code)
.exit_code)

@mock.patch('random.uniform')
def test_number_bounds(self, mock_random):
Expand All @@ -72,37 +73,34 @@ def test_number_bounds(self, mock_random):
# Test inclusive lower bound
target_input["exclusive-minimum"] = False
mock_random.return_value = -0.001
self.assertRaises(SystemExit, bosh.execute, ["simulate",
json.dumps(test_json),
"-j"])
self.assertRaises(ExecutorError, bosh.execute, ("simulate",
json.dumps(test_json),
"-j"))
mock_random.return_value = 0
ret = bosh.bosh(args=["example", json.dumps(test_json)])
self.assertIsInstance(json.loads(ret), dict)

# Test exclusive lower bound
target_input["exclusive-minimum"] = True
self.assertRaises(SystemExit, bosh.execute, ["simulate",
json.dumps(test_json),
"-j"])
self.assertRaises(ExecutorError, bosh.execute,
("simulate", json.dumps(test_json), "-j"))
mock_random.return_value = 0.001
ret = bosh.bosh(args=["example", json.dumps(test_json)])
self.assertIsInstance(json.loads(ret), dict)

# Test inclusive upper bound
target_input["exclusive-maximum"] = False
mock_random.return_value = 1.001
self.assertRaises(SystemExit, bosh.execute, ["simulate",
json.dumps(test_json),
"-j"])
self.assertRaises(ExecutorError, bosh.execute,
("simulate", json.dumps(test_json), "-j"))
mock_random.return_value = 1
ret = bosh.bosh(args=["example", json.dumps(test_json)])
self.assertIsInstance(json.loads(ret), dict)

# Test exclusive upper bound
target_input["exclusive-maximum"] = True
self.assertRaises(SystemExit, bosh.execute, ["simulate",
json.dumps(test_json),
"-j"])
self.assertRaises(ExecutorError, bosh.execute,
("simulate", json.dumps(test_json), "-j"))
mock_random.return_value = 0.999
ret = bosh.bosh(args=["example", json.dumps(test_json)])
self.assertIsInstance(json.loads(ret), dict)
Expand All @@ -119,44 +117,32 @@ def test_success_json(self):
def test_failing_bad_descriptor_invo_combos(self):
example1_dir = os.path.join(self.get_examples_dir(), "example1")

self.assertRaises(SystemExit, bosh.execute, ["simulate",
os.path.join(example1_dir,
"fake.json"),
"-i",
os.path.join(
example1_dir,
"invocation.json")])
self.assertRaises(SystemExit, bosh.execute, ["simulate",
os.path.join(
example1_dir,
"example1_docker.json"),
"-i",
os.path.join(example1_dir,
"fake.json")])
self.assertRaises(SystemExit, bosh.execute, ["simulate",
os.path.join(
example1_dir,
"example1_docker.json"),
"-i",
os.path.join(
example1_dir,
"exampleTool1.py")])
self.assertRaises(SystemExit, bosh.execute, ["simulate",
os.path.join(
example1_dir,
"example1_docker.json")])
self.assertRaises(SystemExit, bosh.execute, ["simulate",
os.path.join(
example1_dir,
"example1_docker.json"),
"-i",
os.path.join(
example1_dir,
"invocation.json")])
self.assertRaises(SystemExit, bosh.execute, ["simulate",
os.path.join(
example1_dir,
"example1_docker.json")])
self.assertRaises(ExecutorError, bosh.execute,
("simulate", os.path.join(example1_dir, "fake.json"),
"-i", os.path.join(example1_dir, "invocation.json")))

self.assertRaises(ExecutorError, bosh.execute,
("simulate", os.path.join(example1_dir,
"example1_docker.json"),
"-i", os.path.join(example1_dir, "fake.json")))

self.assertRaises(ExecutorError, bosh.execute,
("simulate", os.path.join(example1_dir,
"example1_docker.json"),
"-i", os.path.join(example1_dir, "exampleTool1.py")))

self.assertRaises(ExecutorError, bosh.execute,
("simulate", os.path.join(example1_dir,
"example1_docker.json")))

self.assertRaises(ExecutorError, bosh.execute,
("simulate", os.path.join(example1_dir,
"example1_docker.json"),
"-i", os.path.join(example1_dir, "invocation.json")))

self.assertRaises(ExecutorError, bosh.execute,
("simulate", os.path.join(example1_dir,
"example1_docker.json")))

def test_collapsing_whitespace_optionals(self):
descriptor = os.path.join(os.path.split(bfile)[0],
Expand Down
13 changes: 13 additions & 0 deletions tools/python/boutiques/util/utils.py
View file
Expand Up @@ -169,3 +169,16 @@ def camelCaseInputIds(descriptor):
"\'{0}\'".format(v))
descriptor = json.loads(plainTextDesc)
return descriptor


def formatSphinxUsage(func, usage_str):
args = usage_str.replace("[", " ")\
.replace("]", " ")\
.replace("\n", "")\
.split(func)[1:]
args = "".join(args)
args = args.split(" ")[0:]
args = list(filter(lambda x: x != "", args))
args = ["\"{}\"".format(arg.strip()) for arg in args]
args = ", ".join(args)
return "[{}]".format(args)

0 comments on commit 4d4bb27

Please sign in to comment.