/
cli.py
173 lines (145 loc) · 8.58 KB
/
cli.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
""" Methods for test cases involving checking command-line interfaces
:Author: Jonathan Karr <karr@mssm.edu>
:Date: 2020-12-21
:Copyright: 2020, Center for Reproducible Biomedical Modeling
:License: MIT
"""
from ..data_model import TestCase
from ..warnings import TestCaseWarning
from biosimulators_utils.simulator.environ import ENVIRONMENT_VARIABLES
import re
import subprocess
import warnings
__all__ = [
'CliDisplaysHelpInline',
'CliDescribesSupportedEnvironmentVariablesInline',
'CliDisplaysVersionInformationInline',
]
class CliDisplaysHelpInline(TestCase):
""" Test that a command-line interface provides inline help. """
def eval(self, specifications, working_dirname, synthetic_archives_dir=None, dry_run=False, cli=None):
""" Evaluate a simulator's performance on a test case
Args:
specifications (:obj:`dict`): specifications of the simulator to validate
working_dirname (:obj:`str`): directory for temporary files for evaluating test case
synthetic_archives_dir (:obj:`str`, optional): Directory to save the synthetic COMBINE/OMEX archives
generated by the test cases
dry_run (:obj:`bool`): if :obj:`True`, do not use the simulator to execute COMBINE/OMEX archives.
cli (:obj:`str`, optional): command-line interface to use to execute the tests involving the simulation of COMBINE/OMEX
archives rather than a Docker image
Raises:
:obj:`Exception`: if the simulator did not pass the test case
"""
self.get_simulator_docker_image(specifications)
image_url = specifications['image']['url']
cli = [cli] if cli else ['docker', 'run', '--tty', '--rm', image_url]
result = subprocess.run(cli, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
log = result.stdout.decode() if result.stdout else ''
supported = (
'-i' in log
and '-o' in log
)
if not supported:
warnings.warn(('Command-line interfaces should display basic help when no arguments are provided.\n\n'
'The command-line interface displayed the following when no argument was provided:\n\n {}'
).format(log.replace('\n', '\n ')),
TestCaseWarning)
result = subprocess.run(cli + ['-h'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
log = result.stdout.decode() if result.stdout else ''
supported = (
'arguments' in log
and '-i' in log
and '--archive' in log
and '-o' in log
and '--out-dir' in log
)
if not supported:
warnings.warn(('Command-line interface should support the `-h` option for displaying help inline.\n\n'
'The command-line interface displayed the following when executed with `-h`:\n\n {}'
).format(log.replace('\n', '\n ')),
TestCaseWarning)
result = subprocess.run(cli + ['--help'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
log = result.stdout.decode() if result.stdout else ''
supported = (
'arguments' in log
and '-i' in log
and '--archive' in log
and '-o' in log
and '--out-dir' in log
)
if not supported:
warnings.warn(('Command-line interface should support the `--help` option for displaying help inline.\n\n'
'The command-line interface displayed the following when executed with `--help`:\n\n {}'
).format(log.replace('\n', '\n ')),
TestCaseWarning)
class CliDescribesSupportedEnvironmentVariablesInline(TestCase):
""" Test that the inline help for a command-line interface describes the environment variables that the
simulator supports.
"""
def eval(self, specifications, working_dirname, synthetic_archives_dir=None, dry_run=False, cli=None):
""" Evaluate a simulator's performance on a test case
Args:
specifications (:obj:`dict`): specifications of the simulator to validate
working_dirname (:obj:`str`): directory for temporary files for evaluating test case
synthetic_archives_dir (:obj:`str`, optional): Directory to save the synthetic COMBINE/OMEX archives
generated by the test cases
dry_run (:obj:`bool`): if :obj:`True`, do not use the simulator to execute COMBINE/OMEX archives.
cli (:obj:`str`, optional): command-line interface to use to execute the tests involving the simulation of COMBINE/OMEX
archives rather than a Docker image
Raises:
:obj:`Exception`: if the simulator did not pass the test case
"""
self.get_simulator_docker_image(specifications)
image_url = specifications['image']['url']
cli = [cli] if cli else ['docker', 'run', '--tty', '--rm', image_url]
result = subprocess.run(cli + ['-h'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
log = result.stdout.decode() if result.stdout else ''
potentially_missing_env_vars = []
for var in ENVIRONMENT_VARIABLES.values():
if var.name not in log:
potentially_missing_env_vars.append(var.name)
if potentially_missing_env_vars:
msg = ('The inline help for a command-line interface for a simulation tool should describe the '
'environment variables that the simulation tool supports.\n\n'
'The command-line interface does not describe the following standard environment '
'variables recognized by BioSimulators:\n - {}\n\n'
'If the simulation tool implements these variables, they should be described in the inline help for '
'its command-line interface.\n\n'
'Note, support for these environment variables is optional. Simulation tools are not required to support '
'these variables.'
).format('\n - '.join("'" + var + "'" for var in sorted(potentially_missing_env_vars)))
warnings.warn(msg, TestCaseWarning)
class CliDisplaysVersionInformationInline(TestCase):
""" Test that a command-line interface provides version information inline. """
def eval(self, specifications, working_dirname, synthetic_archives_dir=None, dry_run=False, cli=None):
""" Evaluate a simulator's performance on a test case
Args:
specifications (:obj:`dict`): specifications of the simulator to validate
working_dirname (:obj:`str`): directory for temporary files for evaluating test case
synthetic_archives_dir (:obj:`str`, optional): Directory to save the synthetic COMBINE/OMEX archives
generated by the test cases
dry_run (:obj:`bool`): if :obj:`True`, do not use the simulator to execute COMBINE/OMEX archives.
cli (:obj:`str`, optional): command-line interface to use to execute the tests involving the simulation of COMBINE/OMEX
archives rather than a Docker image
Raises:
:obj:`Exception`: if the simulator did not pass the test case
"""
self.get_simulator_docker_image(specifications)
image_url = specifications['image']['url']
cli = [cli] if cli else ['docker', 'run', '--tty', '--rm', image_url]
result = subprocess.run(cli + ['-v'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
log = result.stdout.decode() if result.stdout else ''
supported = re.search(r'\d+\.\d+', log)
if not supported:
warnings.warn(('Command-line interface should support the `-v` option for displaying version information inline.\n\n'
'The command-line interface displayed the following when executed with `-v`:\n\n {}'
).format(log.replace('\n', '\n ')),
TestCaseWarning)
result = subprocess.run(cli + ['--version'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
log = result.stdout.decode() if result.stdout else ''
supported = re.search(r'\d+\.\d+', log)
if not supported:
warnings.warn(('Command-line interface should support the `--version` option for displaying version information inline.\n\n'
'The command-line interface displayed the following when executed with `--version`:\n\n {}'
).format(log.replace('\n', '\n ')),
TestCaseWarning)