Merge pull request #169 from pagreene/make-cli

Make CLI's

benchmarker/cli.py

@@ -0,0 +1,147 @@
+import os
+import subprocess
+import webbrowser
+
+from numpy import array
+from datetime import datetime
+from collections import defaultdict
+from typing import Iterable
+
+import click
+
+from benchmarker.util import benchmark, list_apis, list_stacks, save_results
+
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+
+
+@click.group()
+def main():
+    """The benchmarker CLI.
+
+    The benchmarker tool allows stack deployments to be
+    compared based on the time taken to run existing test corpora that utilize
+    the web service.
+    """
+
+
+@main.command('list')
+@click.argument("list_scope", type=click.Choice(["apis", "stacks"]),
+                required=False)
+def print_list(list_scope):
+    """List the apis or stacks that have already been used."""
+    def print_apis():
+        print()
+        print("Existing API Test Corpora")
+        print("-------------------------")
+        for api in list_apis():
+            print(api)
+
+    def print_stacks():
+        print()
+        print("Existing Tested Stacks")
+        print("----------------------")
+        for stack_name in list_stacks():
+            print(stack_name)
+
+    if list_scope == 'apis':
+        print_apis()
+    elif list_scope == 'stacks':
+        print_stacks()
+    else:
+        print_apis()
+        print_stacks()
+
+
+@main.command()
+@click.argument("test_corpus")
+@click.argument("stack_name")
+@click.argument("api_name")
+@click.option("-r", "--inner-runs", default=1,
+              type=click.IntRange(1, 100),
+              help="Select the number of times to repeat the test in a row.")
+@click.option("-R", "--outer-runs", default=1,
+              type=click.IntRange(1, 100),
+              help=("Select the number of times to repeat the entire suite of "
+                    "tests."))
+def run(test_corpus, stack_name, api_name, inner_runs, outer_runs):
+    """Run the benchmarker and save the aggregate the results.
+
+    \b
+    The TEST_CORPUS should be a path to a python test file that tests the INDRA
+    Database REST service, using the standard convention:
+
+        "path/to/test_file.py:test_function"
+
+    The STACK_NAME should name a readonly-build stack (database and service
+    deployment) that are being tested. You can get a list of existing
+    (previously tested) stacks using `indra_db_benchmarker list`.
+
+    The API_NAME should give a name for the test corpus that is being used. You
+    can get a list of existing (previously used) corpora using the `list`
+    feature.
+    """
+    import tabulate
+    start_time = datetime.utcnow()
+
+    # Run the benchmarker. Run it `outer_run` times, and we will aggregate
+    # the results below.
+    result_list = []
+    test_names = []
+    for i in range(outer_runs):
+        run_result = benchmark(test_corpus, num_runs=inner_runs)
+        if not test_names:
+            test_names = list(run_result.keys())
+        result_list.append(run_result)
+
+    # Aggregate the results from above, either adding values to the list
+    # or extending a list.
+    results = {}
+    for test_name in test_names:
+        test_results = defaultdict(list)
+        for this_result in result_list:
+            test_data = this_result[test_name]
+            for data_name, data_val in test_data.items():
+                if isinstance(data_val, Iterable):
+                    test_results[data_name].extend(data_val)
+                else:
+                    test_results[data_name].append(data_val)
+
+        # Convert the default dict into a real dict.
+        test_results = dict(test_results)
+
+        # Turn the time data into an array, and calculate mean and std dev.
+        time_data = array(test_results['times'])
+        test_results['duration'] = time_data.mean()
+        test_results['deviation'] = time_data.std()
+
+        # Calculate the overall pass rate.
+        test_results['passed'] = sum(test_results['passed'])/outer_runs
+
+        # Add this test's aggregated results to the results object.
+        results[test_name] = test_results
+
+    rows = [(test, st['passed'], st['duration'], st['deviation'])
+            for test, st in results.items()]
+    headers = ('Test', 'Fraction Passed', 'Ave. Duration', 'Std. Deviation')
+    print(tabulate.tabulate(rows, headers))
+    save_results(start_time, api_name, stack_name, results)
+
+
+@main.command()
+def view():
+    """Run the web service to view results."""
+    basic_env = os.environ.copy()
+    basic_env['FLASK_APP'] = os.path.join(HERE, "viewer_app/app.py:app")
+    print("Starting web server...")
+    p = subprocess.Popen(['flask', 'run', '--port', '5280'],
+                         env=basic_env, stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE)
+    print("Opening browser...")
+    webbrowser.open("http://localhost:5280")
+    print("Press Ctrl-C to exit.")
+    p.wait()
+
+
+if __name__ == "__main__":
+    main()

benchmarker/util.py

@@ -54,52 +54,66 @@ def run_test(test_name, test_func, num_runs):
     return test_results
 
 
-def benchmark(loc, base_name=None, num_runs=1):
+def benchmark(test_selection=None, base_name=None, num_runs=1):
+    """Run a benchmark of the REST service using a given test corpus.
+
+    Parameters
+    ----------
+    test_selection : Optional[str]
+        Specify the location of the test or tests you wish to run, using the
+        standard formalism: "path/to/test.py:specific_test", where any less
+        specification will result in a search for things that start with "test_"
+        recursively, as usual.
+    base_name : Optional[str]
+        Give this benchmark a base name.
+    num_runs : Optional[int]
+        Specify how many times the tests should be run.
+    """
     # By default, just run in this directory
-    if loc is None:
-        loc = os.path.abspath('.')
+    if test_selection is None:
+        test_selection = os.path.abspath('.')
 
     # Extract a function name, if it was included.
-    if loc.count(':') == 0:
+    if test_selection.count(':') == 0:
         func_name = None
-    elif loc.count(':') == 1:
-        loc, func_name = loc.split(':')
+    elif test_selection.count(':') == 1:
+        test_selection, func_name = test_selection.split(':')
     else:
-        raise ValueError(f"Invalid loc: {loc}")
-    mod_name = os.path.basename(loc).replace('.py', '')
+        raise ValueError(f"Invalid loc: {test_selection}")
+    mod_name = os.path.basename(test_selection).replace('.py', '')
     if base_name:
         mod_name = base_name + '.' + mod_name
 
     # Check if the location exists, and whether it is a directory or file.
     # Handle the file case by recursively calling this function for each file.
     results = {}
-    if not os.path.exists(loc):
-        raise ValueError(f"No such file or directory: {loc}")
-    elif os.path.isdir(loc):
+    if not os.path.exists(test_selection):
+        raise ValueError(f"No such file or directory: {test_selection}")
+    elif os.path.isdir(test_selection):
         if func_name is not None:
             raise ValueError("To specify function, location must be a file.")
-        for file in os.listdir(loc):
-            new_path = os.path.join(loc, file)
+        for file in os.listdir(test_selection):
+            new_path = os.path.join(test_selection, file)
             if ('test' in file and os.path.isfile(new_path)
                     and new_path.endswith('.py')):
                 results.update(benchmark(new_path, base_name=mod_name,
                                          num_runs=num_runs))
         return results
 
     # Handle the case a file is specified.
-    if not loc.endswith('.py'):
-        raise ValueError(f"Location {loc} is not a python file.")
-    print("="*len(loc))
-    print(loc)
-    print('-'*len(loc))
-    spec = spec_from_file_location(mod_name, loc)
+    if not test_selection.endswith('.py'):
+        raise ValueError(f"Location {test_selection} is not a python file.")
+    print("=" * len(test_selection))
+    print(test_selection)
+    print('-' * len(test_selection))
+    spec = spec_from_file_location(mod_name, test_selection)
     test_module = module_from_spec(spec)
     try:
         spec.loader.exec_module(test_module)
     except KeyboardInterrupt:
         raise
     except Exception as err:
-        logger.error(f"Failed to load {loc}, skipping...")
+        logger.error(f"Failed to load {test_selection}, skipping...")
         logger.exception(err)
         return results