Merge pull request #120 from pagreene/benchmarker

Create a simple benchmarking tool

benchmarker/__init__.py

@@ -0,0 +1 @@
+from .util import *

benchmarker/benchmark

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+
+from datetime import datetime
+from argparse import ArgumentParser
+
+from benchmarker.util import *
+
+
+def main():
+    start_time = datetime.utcnow()
+    parser = ArgumentParser(
+        description=('Run tests and benchmark time to run and '
+                     'errors.'))
+    subparsers = parser.add_subparsers(
+        dest='task',
+        help="Select the action you wish to perform."
+    )
+
+    # Set up the run parser
+    run_parser = subparsers.add_parser('run', help='Run benchmark tests.')
+    run_parser.add_argument(dest='location',
+                            help='The location of the test file or directory.')
+    run_parser.add_argument(dest='stack_name',
+                            help='The name of the Readonly stack being tested.')
+    run_parser.add_argument(dest='api_name',
+                            help=('The name of the dependant API being tested. '
+                                  'This label is used to group test results on '
+                                  's3. Whenever possible, try to use existing '
+                                  'labels.'))
+
+    # Setup the list parser
+    list_parser = subparsers.add_parser('list', help='List certain properties.')
+    list_parser.add_argument(choices=['apis', 'stacks'], dest='list_scope')
+
+    # Parse the arguments, run the code.
+    args = parser.parse_args()
+    if args.task == 'list':
+        if args.list_scope == 'apis':
+            for api in list_apis():
+                print(api)
+        elif args.list_scope == 'stacks':
+            for stack_name in list_stacks():
+                print(stack_name)
+    elif args.task == 'run':
+        results = benchmark(args.location)
+        for test, stats in results.items():
+            print(test, stats)
+        save_results(start_time, args.api_name, args.stack_name, results)
+
+
+if __name__ == '__main__':
+    main()

benchmarker/util.py

@@ -0,0 +1,128 @@
+__all__ = ['benchmark', 'list_apis', 'list_stacks', 'save_results']
+
+import os
+import json
+
+import boto3
+import logging
+from datetime import datetime
+from inspect import getmembers, isfunction
+from importlib.util import spec_from_file_location, module_from_spec
+
+
+logger = logging.getLogger('benchmark_tools')
+
+BUCKET = 'bigmech'
+PREFIX = 'indra-db/benchmarks/'
+
+
+def benchmark(loc, base_name=None):
+    # By default, just run in this directory
+    if loc is None:
+        loc = os.path.abspath('.')
+
+    # Extract a function name, if it was included.
+    if loc.count(':') == 0:
+        func_name = None
+    elif loc.count(':') == 1:
+        loc, func_name = loc.split(':')
+    else:
+        raise ValueError(f"Invalid loc: {loc}")
+    mod_name = os.path.basename(loc).replace('.py', '')
+    if base_name:
+        mod_name = base_name + '.' + mod_name
+
+    # Check if the location exists, and whether it is a directory or file.
+    # Handle the file case by recursively calling this function for each file.
+    results = {}
+    if not os.path.exists(loc):
+        raise ValueError(f"No such file or directory: {loc}")
+    elif os.path.isdir(loc):
+        if func_name is not None:
+            raise ValueError("To specify function, location must be a file.")
+        for file in os.listdir(loc):
+            new_path = os.path.join(loc, file)
+            if ('test' in file and os.path.isfile(new_path)
+                    and new_path.endswith('.py')):
+                results.update(benchmark(new_path, base_name=mod_name))
+        return results
+
+    # Handle the case a file is specified.
+    if not loc.endswith('.py'):
+        raise ValueError(f"Location {loc} is not a python file.")
+    print("="*len(loc))
+    print(loc)
+    print('-'*len(loc))
+    spec = spec_from_file_location(mod_name, loc)
+    test_module = module_from_spec(spec)
+    try:
+        spec.loader.exec_module(test_module)
+    except KeyboardInterrupt:
+        raise
+    except Exception as err:
+        logger.error(f"Failed to load {loc}, skipping...")
+        logger.exception(err)
+        return results
+
+    # Run tests
+    tests = (f for f, _ in getmembers(test_module, isfunction) if 'test' in f)
+    for test_name in tests:
+        test_results = dict.fromkeys(['passed', 'error_type', 'error_str',
+                                      'duration'])
+        print(test_name)
+        print('-'*len(test_name))
+        print("LOGS:")
+        test = getattr(test_module, test_name)
+        start = datetime.now()
+        try:
+            test()
+            print('-'*len(test_name))
+            print("PASSED!")
+            test_results['passed'] = True
+        except Exception as e:
+            print('-'*len(test_name))
+            print("FAILED!", type(e), e)
+            logger.exception(e)
+            test_results['passed'] = False
+            test_results['error_type'] = str(type(e))
+            test_results['error_str'] = str(e)
+        finally:
+            end = datetime.now()
+            test_results['duration'] = (end - start).total_seconds()
+            print()
+            results[f'{mod_name}.{test_name}'] = test_results
+
+    return results
+
+
+def list_apis():
+    """List the current API names on s3."""
+    s3 = boto3.client('s3')
+    res = s3.list_objects_v2(Bucket=BUCKET, Prefix=PREFIX, Delimiter='/')
+    return [e['Prefix'][len(PREFIX):-1] for e in res['CommonPrefixes']]
+
+
+def list_stacks():
+    """List the stacks represented on s3."""
+    s3 = boto3.client('s3')
+    stack_names = set()
+    for api_name in list_apis():
+        try:
+            api_prefix = f'{PREFIX}{api_name}/'
+            res = s3.list_objects_v2(Bucket=BUCKET, Prefix=api_prefix,
+                                     Delimiter='/')
+            stack_names |= {e['Prefix'][len(api_prefix):-1]
+                            for e in res['CommonPrefixes']}
+        except KeyError:
+            logger.error(f"Failed to inspect {api_prefix}: likely malformed "
+                         f"content was added to s3.")
+            continue
+    return list(stack_names)
+
+
+def save_results(start_time, api_name, stack_name, results):
+    """Save the result of a test on s3."""
+    s3 = boto3.client('s3')
+    data_key = f'{PREFIX}{api_name}/{stack_name}/{start_time}.json'
+    s3.put_object(Bucket=BUCKET, Key=data_key, Body=json.dumps(results))
+    return

benchmarker/viewer_app/app.py

@@ -0,0 +1,53 @@
+import json
+from collections import defaultdict
+
+import boto3
+import logging
+from os import path
+from flask import Flask, jsonify
+
+from benchmarker.util import list_stacks, list_apis
+
+logger = logging.getLogger('benchmark_viewer')
+
+HERE = path.dirname(__file__)
+
+app = Flask('benchmark_viewer')
+BUCKET = 'bigmech'
+PREFIX = 'indra-db/benchmarks/'
+
+
+def load(**kwargs):
+    with open(path.join(HERE, 'benchmark.html'), 'r') as f:
+        s = f.read()
+    for key, value in kwargs.items():
+        s = s.replace(f'{{{{ {key} }}}}', json.dumps(value))
+    return s
+
+
+@app.route('/', methods=['GET'])
+def serve_page():
+    return load(stacks=list_stacks(), apis=list_apis())
+
+
+@app.route('/fetch/<corpus_name>/<stack_name>', methods=['GET'])
+def get_stack_data(corpus_name, stack_name):
+    try:
+        s3 = boto3.client('s3')
+        res = s3.list_objects_v2(Bucket=BUCKET,
+                                 Prefix=f'{PREFIX}{corpus_name}/{stack_name}/')
+        keys = {path.basename(e['Key']).split('.')[0]: e['Key']
+                for e in res['Contents']}
+        sorted_keys = list(sorted(keys.items(), key=lambda t: t[0],
+                                  reverse=True))
+        result = defaultdict(dict)
+        for date_str, key in sorted_keys[:5]:
+            date_str = path.basename(key).split('.')[0]
+            file = s3.get_object(Bucket=BUCKET, Key=key)
+            data = json.loads(file['Body'].read())
+            for test_name, test_data in data.items():
+                result[test_name][date_str] = test_data
+    except Exception as e:
+        logger.exception(e)
+        return jsonify({'message': f'Error: {e}'}), 500
+    return jsonify({'message': 'success', 'tests': result}), 200

benchmarker/viewer_app/benchmark.html

@@ -0,0 +1,140 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <title>INDRA DB Benchmark</title>
+
+  <!-- Vue dev CDN -->
+  <script src="https://cdn.jsdelivr.net/npm/vue/dist/vue.js"></script>
+
+  <!-- Vue Multi-Select -->
+  <script src="https://unpkg.com/vue-multiselect@2.1.0"></script>
+  <link rel="stylesheet" href="https://unpkg.com/vue-multiselect@2.1.0/dist/vue-multiselect.min.css">
+
+  <!-- CSS only -->
+  <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.0/css/bootstrap.min.css" integrity="sha384-9aIt2nRpC12Uk9gS9baDl411NQApFmC26EwAOH8WgZl5MYYxFfc+NcPb1dKGj7Sk" crossorigin="anonymous">
+
+  <!-- JS, Popper.js, and jQuery -->
+  <script src="https://code.jquery.com/jquery-3.5.1.slim.min.js" integrity="sha384-DfXdz2htPH0lsSSs5nCTpuj/zy4C+OGpamoFVy38MVBnE+IbbVYUew+OrCXaRkfj" crossorigin="anonymous"></script>
+  <script src="https://cdn.jsdelivr.net/npm/popper.js@1.16.0/dist/umd/popper.min.js" integrity="sha384-Q6E9RHvbIyZFJoft+2mJbHaEWldlvI9IOYy5n3zV9zzTtmI3UksdQRVvoxMfooAo" crossorigin="anonymous"></script>
+  <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.5.0/js/bootstrap.min.js" integrity="sha384-OgVRvuATP1z7JjHLkuOU7Xw704+h835Lr+6QL9UvYjZE3Ipu6Tp75j7Bh/kR0JKI" crossorigin="anonymous"></script>
+
+</head>
+<body>
+
+<div class="container">
+  <div id="app">
+    <div class="row">
+      <h1>Benchmarks</h1>
+    </div>
+    <div class="row">
+      <div class="col-6">
+        <multiselect v-model="selected_api"
+                     :options="apis"
+                     placeholder="Select API..."></multiselect>
+      </div>
+      <div class="col" v-show="selected_api">
+        <multiselect v-model="selected_stacks"
+                     :options="stacks"
+                     :multiple="true"
+                     :loading="isLoading"
+                     :hide-selected="true"
+                     :clear-on-select="false"
+                     :close-on-select="false"
+                     placeholder="Select stack..."
+                     @select="getData"
+                     @remove="dropData"></multiselect>
+      </div>
+    </div>
+    <div v-for="(stack_tests, stack_name) in tests" :key="stack_name">
+      <h1>{{ stack_name }}</h1>
+      <div v-for="test_name in test_names" class="row" :key="test_name">
+        <div class="col-3">
+          {{ minTestNameMap[test_name] }}
+        </div>
+        <div class="col-1"
+             v-for="(test_run, date_str) in stack_tests[test_name]"
+             :key="date_str"
+             :style="getColor(test_run)">
+          {{ Math.round( (test_run.duration + Number.EPSILON) * 10 ) / 10 }}
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+<script>
+
+  app = new Vue({
+    el: '#app',
+    components: {Multiselect: window.VueMultiselect.default},
+    data: {
+      stacks: {{ stacks }},
+      selected_stacks: [],
+      apis: {{ apis }},
+      selected_api: null,
+      tests: {},
+      isLoading: false,
+      test_names: [],
+    },
+    methods: {
+      getData: async function(added) {
+        console.log(added)
+        console.log(this.selected_stacks);
+        this.isLoading = true;
+        const resp = await fetch(`/fetch/${this.selected_api}/${added}`);
+        const data = await resp.json();
+        console.log(data);
+        this.tests[added] = data.tests;
+        for (let test_name in this.tests[added]) {
+          if (this.test_names.includes(test_name))
+            continue
+          this.test_names.push(test_name);
+        }
+        this.isLoading = false;
+      },
+
+      dropData: function(removed) {
+        console.log('input', removed);
+        console.log('selected', this.selected_stacks);
+        Vue.delete(this.tests, removed);
+      },
+
+      getColor: function(test_res) {
+        let color;
+        let text_color = 'white';
+        if (test_res.passed) {
+          color = 'green';
+        } else if (test_res.error_type === '<class \'unittest.case.SkipTest\'>') {
+          color = 'yellow';
+          text_color = 'black';
+        } else {
+          color = 'red';
+        }
+        return `background-color: ${color}; color: ${text_color};`
+      }
+    },
+    computed: {
+      minTestNameMap: function() {
+        // Check if names is empty
+        if (!this.test_names.length)
+          return {};
+
+        // Get the index of the largest common prefix.
+        let names = this.test_names.concat().sort();
+        let L = names[0].length;
+        let i = 0;
+        while (i < L && names.every(n => n.charAt(i) === names[0].charAt(i))) i++;
+
+        // Get the shortened names.
+        let shortest_names = [];
+        for (let test_name of names)
+          shortest_names[test_name] = test_name.substring(i);
+        return shortest_names;
+      }
+    }
+  });
+</script>
+
+</body>
+</html>