Add new measurer based on Mutation Analysis

.dockerignore

@@ -7,4 +7,6 @@
 .venv
 **__pycache__*
 docs
-report*
+report*
+fuzzers/mutation_analysis/mua_fuzzer_bench/.git
+fuzzers/mutation_analysis/fuzzbench_mapped_dir/

.gitignore

@@ -44,3 +44,7 @@ docker/generated.mk
 
 # Vim backup files.
 .*.swp
+
+# mua related files and directories.
+fuzzers/mutation_analysis/fuzzbench_mapped_dir/
+mua_out/

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "fuzzers/mutation_analysis/mua_fuzzer_bench"]
+	path = fuzzers/mutation_analysis/mua_fuzzer_bench
+	url = https://github.com/phi-go/mua_fuzzer_bench
+	branch = sbft

analysis/benchmark_results.py

@@ -21,6 +21,9 @@
 from analysis import stat_tests
 from common import benchmark_utils
 from common import filestore_utils
+from common import logs
+
+logger = logs.Logger()
 
 
 # pylint: disable=too-many-public-methods, too-many-arguments
@@ -36,13 +39,14 @@ class BenchmarkResults:
     """
 
     def __init__(self, benchmark_name, experiment_df, coverage_dict,
-                 output_directory, plotter):
+                 output_directory, plotter, mua_results):
         self.name = benchmark_name
 
         self._experiment_df = experiment_df
         self._coverage_dict = coverage_dict
         self._output_directory = output_directory
         self._plotter = plotter
+        self.mua_results = mua_results
 
     def _prefix_with_benchmark(self, filename):
         return self.name + '_' + filename
@@ -56,6 +60,28 @@ def get_coverage_report_path(self, fuzzer_name, benchmark_name):
             fuzzer_name, benchmark_name, self._benchmark_df)
         return filestore_utils.get_user_facing_path(filestore_path)
 
+    def get_mua_report_data(self, _fuzzer_name, _benchmark_name):
+        """Returns results as string"""
+        return 'TODO: Not Implemented Yet'  #TODO: implement this or delete
+
+    @property
+    def mutation_analysis_plot(self):
+        """plot for mutation analysis."""
+        plot_filename = self._prefix_with_benchmark('mutation_analysis.svg')
+        if self.mua_results is None:
+            logger.info(
+                'mutation_analysis_plot not rendered, due to missing data')
+            return None
+
+        (num_trials, fuzzer_pds) = self.mua_results
+
+        num_fuzzers = len(fuzzer_pds)
+
+        self._plotter.write_mutation_analysis_plot(
+            fuzzer_pds, num_fuzzers, num_trials,
+            self._get_full_path(plot_filename))
+        return plot_filename
+
     @property
     @functools.lru_cache()
     def type(self):

analysis/experiment_results.py

@@ -59,13 +59,16 @@ def __init__(  # pylint: disable=too-many-arguments
             coverage_dict,
             output_directory,
             plotter,
+            mua_results=None,
             experiment_name=None):
         if experiment_name:
             self.name = experiment_name
         else:
             # Take name from first row.
             self.name = experiment_df.experiment.iloc[0]
 
+        self.mua_results = mua_results
+
         # FuzzBench repo commit hash.
         self.git_hash = None
         if 'git_hash' in experiment_df.columns:
@@ -135,7 +138,7 @@ def benchmarks(self):
             benchmark_results.BenchmarkResults(name, self._experiment_df,
                                                self._coverage_dict,
                                                self._output_directory,
-                                               self._plotter)
+                                               self._plotter, self.mua_results)
             for name in sorted(benchmark_names)
         ]
 

analysis/generate_report.py

@@ -14,8 +14,13 @@
 """Report generator tool."""
 
 import argparse
+import lzma
 import os
 import sys
+import sqlite3
+import tempfile
+
+from collections import defaultdict
 
 import pandas as pd
 
@@ -27,6 +32,7 @@
 from analysis import rendering
 from common import filesystem
 from common import logs
+from common import experiment_utils
 
 logger = logs.Logger()
 
@@ -44,7 +50,7 @@ def get_arg_parser():
     parser.add_argument(
         '-t',
         '--report-type',
-        choices=['default', 'experimental'],
+        choices=['default', 'experimental', 'with_mua'],
         default='default',
         help='Type of the report (which template to use). Default: default.')
     parser.add_argument(
@@ -58,6 +64,11 @@ def get_arg_parser():
         action='store_true',
         default=False,
         help='If set, plots are created faster, but contain less details.')
+    parser.add_argument('-mua',
+                        '--mutation-analysis',
+                        action='store_true',
+                        default=False,
+                        help='If set, mutation analysis report is created.')
     parser.add_argument(
         '--log-scale',
         action='store_true',
@@ -79,6 +90,11 @@ def get_arg_parser():
                         '--fuzzers',
                         nargs='*',
                         help='Names of the fuzzers to include in the report.')
+    parser.add_argument(
+        '-xb',
+        '--experiment-benchmarks',
+        nargs='*',
+        help='Names of the benchmarks to include in the report.')
     parser.add_argument(
         '-cov',
         '--coverage-report',
@@ -142,6 +158,7 @@ def get_experiment_data(experiment_names,
     logger.info('Reading experiment data from db.')
     experiment_df = queries.get_experiment_data(experiment_names,
                                                 main_experiment_benchmarks)
+    # experiment_df.to_csv('/tmp/experiment-data/experiment_data.csv')
     logger.info('Done reading experiment data from db.')
     description = queries.get_experiment_description(main_experiment_name)
     return experiment_df, description
@@ -186,6 +203,164 @@ def modify_experiment_data_if_requested(  # pylint: disable=too-many-arguments
     return experiment_df
 
 
+def normalized_timestamps(timestamps):
+    """Normalize timestamps."""
+    seed_timestamp_file = next(
+        (tt for tt in timestamps if tt[2] == '<seed_entry>'), None)
+    try:
+        min_timestamp_file = min(
+            (tt for tt in timestamps if tt[2] != '<seed_entry>'),
+            key=lambda x: x[3])
+    except ValueError:
+        min_timestamp_file = seed_timestamp_file
+    try:
+        max_timestamp_file = max(
+            (tt for tt in timestamps if tt[2] != '<seed_entry>'),
+            key=lambda x: x[3])
+    except ValueError:
+        max_timestamp_file = seed_timestamp_file
+    min_timestamp = min_timestamp_file[3]
+    max_timestamp = max_timestamp_file[3]
+    timestamps_normalized = {}
+    for _hashname, input_file_id, input_file, timestamp in timestamps:
+        if input_file == '<seed_entry>':
+            timestamps_normalized[input_file_id] = 0
+        else:
+            timestamps_normalized[input_file_id] = timestamp - min_timestamp
+
+    timespan = max_timestamp - min_timestamp
+    return timestamps_normalized, timespan
+
+
+def get_first_covered_killed(results, timestamps_map):
+    """Get first covered and killed mutant."""
+    ordered_inputs = sorted(results, key=lambda x: timestamps_map[x[0]])
+    mut_result_times = defaultdict(lambda: {'seen': None, 'killed': None})
+    for ordered_input in ordered_inputs:
+        input_file_id, mut_id, skipped, killed = ordered_input[:4]
+        if skipped:
+            continue
+        if mut_id not in mut_result_times:
+            mut_result_times[mut_id]['seen'] = timestamps_map[input_file_id]
+        if killed:
+            assert mut_id in mut_result_times
+            if mut_result_times[mut_id]['killed'] is None:
+                mut_result_times[mut_id]['killed'] = timestamps_map[
+                    input_file_id]
+    return mut_result_times
+
+
+def get_timeline(time_covered_killed, timespan, meta):  # pylint: disable=too-many-locals
+    """Create timeline regarding covering and killing of mutants."""
+    if timespan == 0:
+        max_time_base = 1
+    else:
+        max_time_base = 16
+    fuzz_target, benchmark, fuzzer_name, trial_num, cycle = meta
+    normalized_time_elem = timespan / (max_time_base**2)
+    time = 'time'
+    count_seen = 'seen'
+    count_killed = 'killed'
+    res = []
+    for time_base in range(1, max_time_base + 1):
+        time = normalized_time_elem * (time_base**2)
+        count_seen = 0
+        count_killed = 0
+        for _mut_id, times in time_covered_killed.items():
+            if times['seen'] is not None and times['seen'] <= time:
+                count_seen += 1
+            if times['killed'] is not None and times['killed'] <= time:
+                count_killed += 1
+        res.append((fuzz_target, benchmark, fuzzer_name, trial_num, cycle, time,
+                    count_seen, count_killed))
+    return res
+
+
+def load_result_db(res_db_path):
+    """Load result.sqlite database."""
+    with tempfile.NamedTemporaryFile() as tmp_file:
+        with lzma.open(res_db_path) as res_db:
+            tmp_file.write(res_db.read())
+        tmp_file.flush()
+        tmp_file.seek(0)
+        with sqlite3.connect(tmp_file.name) as conn:
+            run_info = conn.execute(
+                'SELECT benchmark, fuzz_target, fuzzer, trial_num FROM run_info'
+            ).fetchall()
+            results = conn.execute('''SELECT
+                    input_file_id,
+                    mut_id,
+                    skipped,
+                    killed,
+                    orig_retcode,
+                    mutant_retcode,
+                    orig_runtime,
+                    mutant_runtime,
+                    orig_timed_out,
+                    mutant_timed_out
+                FROM results''').fetchall()
+            timestamps = conn.execute(
+                '''SELECT hashname, input_file_id, input_file, timestamp
+                 FROM timestamps''').fetchall()
+    return run_info, results, timestamps
+
+
+def get_mua_results(experiment_df):  # pylint: disable=too-many-locals
+    """Get mutation analysis results for each fuzzer in each trial to use in
+    the report."""
+
+    #get relationship between trial_id and benchmark from df
+    trial_dict = experiment_df.set_index('trial_id')['benchmark'].to_dict()
+
+    experiment_data_dir = experiment_utils.get_experiment_filestore_path()
+    results_data_dir = f'{experiment_data_dir}/mua-results/results'
+
+    if not os.path.isdir(results_data_dir):
+        logger.warning('''mua-results/results dir does not exist,
+              stopping mua report creation''')
+        return None
+
+    fuzzer_pds = defaultdict(list)
+
+    for trial in trial_dict.keys():
+        mua_result_db_file =  f'{results_data_dir}/{trial}/' \
+            'results.sqlite.lzma'
+        if not os.path.isfile(mua_result_db_file):
+            logger.debug(
+                'mua_result_db_file does not exist, this is expected ' +
+                'if only median trial is evaluated: '
+                f'{mua_result_db_file}')
+            continue
+        logger.info(f'found mua_result_db_file: {mua_result_db_file}')
+        run_info, results, timestamps = load_result_db(mua_result_db_file)
+        assert len(run_info) == 1
+        benchmark, fuzz_target, fuzzer, trial_num = run_info[0]
+        timestamps_map, timespan = normalized_timestamps(timestamps)
+
+        results = [
+            rr for rr in results if timestamps_map.get(rr[0]) is not None
+        ]
+        time_covered_killed = get_first_covered_killed(results, timestamps_map)
+        meta = fuzz_target, benchmark, fuzzer, trial_num, trial
+        timeline = get_timeline(time_covered_killed, timespan, meta)
+        pd_timeline = pd.DataFrame(timeline,
+                                   columns=[
+                                       'fuzz_target', 'benchmark', 'fuzzer',
+                                       'trial_num', 'cycle', 'time', 'seen',
+                                       'killed'
+                                   ])
+        fuzzer_pds[fuzzer].append(pd_timeline)
+
+    num_trials = None
+    for fuzzer, fuzzer_pd in fuzzer_pds.items():
+        if num_trials is None:
+            num_trials = len(fuzzer_pd)
+        else:
+            assert num_trials == len(fuzzer_pd)
+
+    return (num_trials, fuzzer_pds)
+
+
 # pylint: disable=too-many-arguments,too-many-locals
 def generate_report(experiment_names,
                     report_directory,
@@ -202,7 +377,8 @@ def generate_report(experiment_names,
                     merge_with_clobber=False,
                     merge_with_clobber_nonprivate=False,
                     coverage_report=False,
-                    experiment_benchmarks=None):
+                    experiment_benchmarks=None,
+                    mutation_analysis=False):
     """Generate report helper."""
     if merge_with_clobber_nonprivate:
         experiment_names = (
@@ -231,6 +407,9 @@ def generate_report(experiment_names,
         experiment_df, experiment_names, benchmarks, fuzzers,
         label_by_experiment, end_time, merge_with_clobber)
 
+    # experiment_df.to_csv('/tmp/experiment-data/out.csv')
+
+    #TODO: make this work with a single fuzzer selected
     # Add |bugs_covered| column prior to export.
     experiment_df = data_utils.add_bugs_covered_column(experiment_df)
 
@@ -247,13 +426,19 @@ def generate_report(experiment_names,
             experiment_df)
         logger.info('Finished generating coverage report info.')
 
+    if mutation_analysis:
+        mua_results = get_mua_results(experiment_df)
+    else:
+        mua_results = None
+
     fuzzer_names = experiment_df.fuzzer.unique()
     plotter = plotting.Plotter(fuzzer_names, quick, log_scale)
     experiment_ctx = experiment_results.ExperimentResults(
         experiment_df,
         coverage_dict,
         report_directory,
         plotter,
+        mua_results=mua_results,
         experiment_name=report_name)
 
     template = report_type + '.html'
@@ -286,7 +471,9 @@ def main():
                     from_cached_data=args.from_cached_data,
                     end_time=args.end_time,
                     merge_with_clobber=args.merge_with_clobber,
-                    coverage_report=args.coverage_report)
+                    coverage_report=args.coverage_report,
+                    experiment_benchmarks=args.experiment_benchmarks,
+                    mutation_analysis=args.mutation_analysis)
 
 
 if __name__ == '__main__':