Merge pull request #136 from pagreene/readonly-update

Readonly update and API overhaul (October 2020 Stable release)

benchmarker/benchmark

@@ -1,8 +1,10 @@
 #!/usr/bin/env python
-
+from collections import defaultdict, Iterable
 from datetime import datetime
 from argparse import ArgumentParser
 
+from numpy import array
+
 from benchmarker.util import *
 
 
@@ -27,6 +29,20 @@ def main():
                                   'This label is used to group test results on '
                                   's3. Whenever possible, try to use existing '
                                   'labels.'))
+    run_parser.add_argument('-r',
+                            dest='inner_runs',
+                            default=1,
+                            choices=range(1, 101),
+                            type=int,
+                            help=('Select the number of times to repeat the '
+                                  'test in a row.'))
+    run_parser.add_argument('-R',
+                            dest='outer_runs',
+                            default=1,
+                            choices=range(1, 101),
+                            type=int,
+                            help=('Select the number of times to repeat the '
+                                  'entire suite of tests.'))
 
     # Setup the list parser
     list_parser = subparsers.add_parser('list', help='List certain properties.')
@@ -35,16 +51,50 @@ def main():
     # Parse the arguments, run the code.
     args = parser.parse_args()
     if args.task == 'list':
+        # Just list some information.
         if args.list_scope == 'apis':
             for api in list_apis():
                 print(api)
         elif args.list_scope == 'stacks':
             for stack_name in list_stacks():
                 print(stack_name)
     elif args.task == 'run':
-        results = benchmark(args.location)
+        # Run the benchmarker. Run it `outer_run` times, and we will aggregate
+        # the results below.
+        result_list = []
+        for i in range(args.outer_runs):
+            run_result = benchmark(args.location, num_runs=args.inner_runs)
+            result_list.append(run_result)
+
+        # Aggregate the results from above, either adding values to the list
+        # or extending a list.
+        results = {}
+        for test_name in run_result.keys():
+            test_results = defaultdict(list)
+            for this_result in result_list:
+                test_data = this_result[test_name]
+                for data_name, data_val in test_data.items():
+                    if isinstance(data_val, Iterable):
+                        test_results[data_name].extend(data_val)
+                    else:
+                        test_results[data_name].append(data_val)
+
+            # Convert the default dict into a real dict.
+            test_results = dict(test_results)
+
+            # Turn the time data into an array, and calculate mean and std dev.
+            time_data = array(test_results['times'])
+            test_results['duration'] = time_data.mean()
+            test_results['deviation'] = time_data.std()
+
+            # Calculate the overall pass rate.
+            test_results['passed'] = sum(test_results['passed'])/args.outer_runs
+
+            # Add this test's aggregated results to the results object.
+            results[test_name] = test_results
+
         for test, stats in results.items():
-            print(test, stats)
+            print(test, stats['passed'], stats['duration'], stats['deviation'])
         save_results(start_time, args.api_name, args.stack_name, results)
 
 

benchmarker/util.py

@@ -6,17 +6,55 @@
 import boto3
 import logging
 from datetime import datetime
-from inspect import getmembers, isfunction
+from inspect import getmembers, isfunction, isclass, ismethod
 from importlib.util import spec_from_file_location, module_from_spec
 
+from numpy import array
+
 
 logger = logging.getLogger('benchmark_tools')
 
 BUCKET = 'bigmech'
 PREFIX = 'indra-db/benchmarks/'
 
 
-def benchmark(loc, base_name=None):
+def run_test(test_name, test_func, num_runs):
+    test_results = dict.fromkeys(['passed', 'error_type', 'error_str',
+                                  'duration', 'deviation', 'times'])
+    test_results['passed'] = False
+    test_results['error_type'] = [None]*num_runs
+    test_results['error_str'] = [None]*num_runs
+    print(test_name)
+    print('-' * len(test_name))
+    durations = []
+    for i in range(num_runs):
+        print("LOGS:")
+        start = datetime.now()
+        try:
+            test_func()
+            print('-' * len(test_name))
+            print("PASSED!")
+            test_results['passed'] += True
+        except Exception as e:
+            print('-' * len(test_name))
+            print("FAILED!", type(e), e)
+            logger.exception(e)
+            test_results['passed'] += False
+            test_results['error_type'][i] = str(type(e))
+            test_results['error_str'][i] = str(e)
+        finally:
+            end = datetime.now()
+            durations.append((end - start).total_seconds())
+            print()
+    dur_array = array(durations)
+    test_results['times'] = durations
+    test_results['duration'] = dur_array.mean()
+    test_results['deviation'] = dur_array.std()
+    test_results['passed'] = test_results['passed'] / num_runs
+    return test_results
+
+
+def benchmark(loc, base_name=None, num_runs=1):
     # By default, just run in this directory
     if loc is None:
         loc = os.path.abspath('.')
@@ -44,7 +82,8 @@ def benchmark(loc, base_name=None):
             new_path = os.path.join(loc, file)
             if ('test' in file and os.path.isfile(new_path)
                     and new_path.endswith('.py')):
-                results.update(benchmark(new_path, base_name=mod_name))
+                results.update(benchmark(new_path, base_name=mod_name,
+                                         num_runs=num_runs))
         return results
 
     # Handle the case a file is specified.
@@ -64,33 +103,31 @@ def benchmark(loc, base_name=None):
         logger.exception(err)
         return results
 
-    # Run tests
-    tests = (f for f, _ in getmembers(test_module, isfunction) if 'test' in f)
+    # Run test functions
+    tests = [f for f, _ in getmembers(test_module, isfunction) if 'test' in f]
     for test_name in tests:
-        test_results = dict.fromkeys(['passed', 'error_type', 'error_str',
-                                      'duration'])
-        print(test_name)
-        print('-'*len(test_name))
-        print("LOGS:")
         test = getattr(test_module, test_name)
-        start = datetime.now()
-        try:
-            test()
-            print('-'*len(test_name))
-            print("PASSED!")
-            test_results['passed'] = True
-        except Exception as e:
-            print('-'*len(test_name))
-            print("FAILED!", type(e), e)
-            logger.exception(e)
-            test_results['passed'] = False
-            test_results['error_type'] = str(type(e))
-            test_results['error_str'] = str(e)
-        finally:
-            end = datetime.now()
-            test_results['duration'] = (end - start).total_seconds()
-            print()
-            results[f'{mod_name}.{test_name}'] = test_results
+        results[f'{mod_name}.{test_name}'] = run_test(test_name, test, num_runs)
+
+    # Run test classes
+    test_classes = [c for c, _ in getmembers(test_module, isclass)
+                    if c.lower().startswith('test')]
+    for class_name in test_classes:
+        cls = getattr(test_module, class_name)
+        obj = cls()
+        test_methods = [m for m, _ in getmembers(obj, ismethod)
+                        if m.lower().startswith('test')
+                        or m.lower() == 'run_test']
+        for method_name in test_methods:
+            obj.setUp()
+            test = getattr(obj, method_name)
+            if method_name == 'run_test' and len(test_methods) == 1:
+                results[f'{mod_name}.{class_name}'] = \
+                    run_test(class_name, test, num_runs)
+            else:
+                results[f'{mod_name}.{class_name}.{method_name}'] = \
+                    run_test(method_name, test, num_runs)
+            obj.tearDown()
 
     return results
 

benchmarker/viewer_app/app.py

@@ -1,5 +1,4 @@
 import json
-from collections import defaultdict
 
 import boto3
 import logging
@@ -30,24 +29,35 @@ def serve_page():
     return load(stacks=list_stacks(), apis=list_apis())
 
 
-@app.route('/fetch/<corpus_name>/<stack_name>', methods=['GET'])
-def get_stack_data(corpus_name, stack_name):
+@app.route('/fetch/<corpus_name>/<stack_name>/<test_file>', methods=['GET'])
+def get_stack_data(corpus_name, stack_name, test_file):
     try:
         s3 = boto3.client('s3')
-        res = s3.list_objects_v2(Bucket=BUCKET,
-                                 Prefix=f'{PREFIX}{corpus_name}/{stack_name}/')
-        keys = {path.basename(e['Key']).split('.')[0]: e['Key']
-                for e in res['Contents']}
-        sorted_keys = list(sorted(keys.items(), key=lambda t: t[0],
-                                  reverse=True))
-        result = defaultdict(dict)
-        for date_str, key in sorted_keys[:5]:
-            date_str = path.basename(key).split('.')[0]
-            file = s3.get_object(Bucket=BUCKET, Key=key)
-            data = json.loads(file['Body'].read())
-            for test_name, test_data in data.items():
-                result[test_name][date_str] = test_data
+        file = s3.get_object(
+            Bucket=BUCKET,
+            Key=f'{PREFIX}{corpus_name}/{stack_name}/{test_file}'
+        )
+        data = json.loads(file['Body'].read())
     except Exception as e:
         logger.exception(e)
         return jsonify({'message': f'Error: {e}'}), 500
-    return jsonify({'message': 'success', 'tests': result}), 200
+    return jsonify({'message': 'success', 'tests': data}), 200
+
+
+@app.route('/list/<corpus_name>', methods=['GET'])
+def list_corpus_options(corpus_name):
+    option_dict = {}
+    try:
+        s3 = boto3.client('s3')
+        prefix = f'{PREFIX}{corpus_name}/'
+        res = s3.list_objects_v2(Bucket=BUCKET, Prefix=prefix)
+        keys = [e['Key'][len(prefix):] for e in res['Contents']]
+        for key in keys:
+            stack, test = key.split('/')
+            test_time = test.split('.')[0]
+            label = f'{test_time} ({stack})'
+            option_dict[label] = {'stack': stack, 'test': test}
+    except Exception as e:
+        logger.exception(e)
+        return jsonify({'message': f'Error: {e}'}), 500
+    return jsonify({'message': 'success', 'options': option_dict})