Merge pull request #808 from tlsfuzzer/friedman-bg

Friedman test in background

tests/test_tlsfuzzer_analysis.py

@@ -17,6 +17,7 @@
     from tlsfuzzer.analysis import Analysis, main, TestPair, help_msg
     import pandas as pd
     import numpy as np
+    import multiprocessing as mp
 except ImportError:
     failed_import = True
 
@@ -99,6 +100,9 @@ def test_report_neq(self):
         timings = pd.DataFrame(data=self.neq_data)
         mock_read_csv = mock.Mock()
         mock_read_csv.return_value = timings
+        def mock_friedman(self, result):
+            result.put(0)
+
         with mock.patch("tlsfuzzer.analysis.Analysis.load_data", mock_read_csv):
             with mock.patch("tlsfuzzer.analysis.Analysis.ecdf_plot") as mock_ecdf:
                 with mock.patch("tlsfuzzer.analysis.Analysis.diff_ecdf_plot") as mock_diff_ecdf:
@@ -107,10 +111,9 @@ def test_report_neq(self):
                             with mock.patch("tlsfuzzer.analysis.Analysis.diff_scatter_plot"):
                                 with mock.patch("tlsfuzzer.analysis.Analysis.conf_interval_plot") as mock_conf_int:
                                     with mock.patch("tlsfuzzer.analysis.Analysis.graph_worst_pair"):
-                                        with mock.patch("tlsfuzzer.analysis.Analysis.friedman_test") as mock_friedman:
+                                        with mock.patch("tlsfuzzer.analysis.Analysis.friedman_test", mock_friedman):
                                             with mock.patch("__main__.__builtins__.open", mock.mock_open()) as mock_open:
                                                 with mock.patch("builtins.print"):
-                                                    mock_friedman.return_value = 0
                                                     analysis = Analysis("/tmp")
                                                     ret = analysis.generate_report()
 
@@ -408,6 +411,55 @@ def test__cent_tend_of_random_sample_with_no_reps(self):
                 self.assertEqual(vals, [])
 
 
+@unittest.skipIf(failed_import,
+                 "Could not import analysis. Skipping related tests.")
+class TestFriedmanNegative(unittest.TestCase):
+    def setUp(self):
+        data = {
+            'A': np.random.normal(size=1000),
+            'B': np.random.normal(size=1000),
+            'C': np.random.normal(size=1000)
+        }
+        timings = pd.DataFrame(data=data)
+        mock_read_csv = mock.Mock()
+        mock_read_csv.return_value = timings
+        with mock.patch("tlsfuzzer.analysis.Analysis.load_data", mock_read_csv):
+            self.analysis = Analysis("/tmp")
+        self.analysis.load_data = mock_read_csv
+
+    def test_friedman_negative(self):
+        friedman_result = mp.Queue()
+        self.analysis.friedman_test(friedman_result)
+
+        result = friedman_result.get()
+
+        self.assertTrue(result > 1e-6)
+
+
+@unittest.skipIf(failed_import,
+                 "Could not import analysis. Skipping related tests.")
+class TestFriedmanInvalid(unittest.TestCase):
+    def setUp(self):
+        data = {
+            'A': np.random.normal(size=10),
+            'B': np.random.normal(size=10),
+        }
+        timings = pd.DataFrame(data=data)
+        mock_read_csv = mock.Mock()
+        mock_read_csv.return_value = timings
+        with mock.patch("tlsfuzzer.analysis.Analysis.load_data", mock_read_csv):
+            self.analysis = Analysis("/tmp")
+        self.analysis.load_data = mock_read_csv
+
+    def test_friedman_negative(self):
+        friedman_result = mp.Queue()
+        self.analysis.friedman_test(friedman_result)
+
+        result = friedman_result.get()
+
+        self.assertIsNone(result)
+
+
 @unittest.skipIf(failed_import,
                  "Could not import analysis. Skipping related tests.")
 class TestPlots(unittest.TestCase):

tlsfuzzer/analysis.py

@@ -307,7 +307,7 @@ def sign_test(self, med=0.0, alternative="two-sided"):
         """
         return self.mt_process(self._sign_test, (med, alternative))
 
-    def friedman_test(self):
+    def friedman_test(self, result):
         """
         Test all classes using Friedman chi-square test.
 
@@ -316,10 +316,11 @@ def friedman_test(self):
         """
         data = self.load_data()
         if len(self.class_names) < 3:
-            return 1
+            result.put(None)
+            return
         _, pval = stats.friedmanchisquare(
             *(data.iloc[:, i] for i in range(len(self.class_names))))
-        return pval
+        result.put(pval)
 
     def _calc_percentiles(self):
         data = self.load_data()
@@ -1103,6 +1104,13 @@ def generate_report(self):
 
         :return: int 0 if no difference was detected, 1 otherwise
         """
+        # the Friedman test is fairly long running, non-multithreadable
+        # and with fairly limited memory use, so run it in background
+        # unconditionally
+        friedman_result = mp.Queue()
+        friedman_process = mp.Process(target=self.friedman_test,
+                                      args=(friedman_result, ))
+        friedman_process.start()
         # plot in separate processes so that the matplotlib memory leaks are
         # not cumulative, see https://stackoverflow.com/q/28516828/462370
         processes = []
@@ -1131,17 +1139,19 @@ def generate_report(self):
 
         self._write_sample_stats()
 
-        friedman_result = self.friedman_test()
-
         difference, p_vals, sign_p_vals, worst_pair, worst_p = \
             self._write_individual_results()
 
         self.graph_worst_pair(worst_pair)
 
+        friedman_process.join()
+
         difference = self._write_summary(difference, p_vals, sign_p_vals,
                                          worst_pair,
-                                         worst_p, friedman_result)
+                                         worst_p, friedman_result.get())
 
+        friedman_result.close()
+        friedman_result.join_thread()
         self._stop_all_threads(processes)
 
         return difference