Closes #16: Apply TTests to all control/variant pairs.

mozilla · Feb 22, 2018 · 580c730 · 580c730
1 parent 9444eb1
commit 580c730
Show file tree

Hide file tree

Showing 5 changed files with 49 additions and 33 deletions.
diff --git a/stmoab/ExperimentDashboard.py b/stmoab/ExperimentDashboard.py
@@ -76,8 +76,10 @@ def _get_event_title_description(self, template, event):
 
     self._params["event"] = event
     self._params["event_string"] = event_string
-    title = description = self._get_title(template["name"]).replace(
-        "Event", event_name)
+    title = description = self._get_title(
+        template["name"]).replace(
+        "Event", event_name).replace(
+        "Scalar_parent_browser_engagement_", "")
 
     if template["description"]:
       description = template["description"].lower().replace(

diff --git a/stmoab/StatisticalDashboard.py b/stmoab/StatisticalDashboard.py
@@ -107,29 +107,35 @@ def _get_ttable_data_for_query(self, label, query_string,
     data = self._get_query_results(query_string, data_source_id, label)
 
     if data is None or len(data) <= 3 or (column_name not in data[0]):
-      return {}
+      return []
 
     control_vals = []
-    exp_vals = []
+    exp_vals = {}
     for row in data:
-      if "type" in row and row["type"].find("control") == -1:
-        exp_vals.append(row[column_name])
-      elif "type" in row:
+      if "type" in row and row["type"].lower().find("control") != -1:
         control_vals.append(row[column_name])
-      else:
-        return {}
+      elif "type" in row:
+        if row["type"] not in exp_vals:
+          exp_vals[row["type"]] = []
 
-    results = self._power_and_ttest(control_vals, exp_vals)
-    return {
-        "Metric": label,
-        "Alpha Error": self.ALPHA_ERROR,
-        "Power": results["power"],
-        "Two-Tailed P-value (ttest)": results["p_val"],
-        "Control Mean": results["control_mean"],
-        "Experiment Mean - Control Mean": results["mean_diff"],
-        "Percent Difference in Means": results["percent_diff"],
-        "Significance": results["significance"]
-    }
+        exp_vals[row["type"]].append(row[column_name])
+      else:
+        return []
+
+    ttable_results = []
+    for variant in exp_vals:
+      results = self._power_and_ttest(control_vals, exp_vals[variant])
+      ttable_results.append({
+          "Metric": "[control vs. {variant}] {metric}".format(variant=variant, metric=label),
+          "Alpha Error": self.ALPHA_ERROR,
+          "Power": results["power"],
+          "Two-Tailed P-value (ttest)": results["p_val"],
+          "Control Mean": results["control_mean"],
+          "Experiment Mean - Control Mean": results["mean_diff"],
+          "Percent Difference in Means": results["percent_diff"],
+          "Significance": results["significance"]
+      })
+    return ttable_results
 
   def _apply_ttable_event_template(self, template, chart_data, events_list,
                                    events_table, title):
@@ -155,20 +161,20 @@ def _apply_ttable_event_template(self, template, chart_data, events_list,
           options
       )
 
-      ttable_row = self._get_ttable_data_for_query(
+      ttable_rows = self._get_ttable_data_for_query(
           event_data["title"],
           query_string,
           "count",
           template["data_source_id"])
 
-      if len(ttable_row) == 0:
+      if len(ttable_rows) == 0:
         self._logger.info((
             "StatisticalDashboard: "
             "Query '{name}' has no relevant data and will not be "
             "included in T-Table.".format(name=event_data["title"])))
         continue
 
-      self._ttables[title]["rows"].append(ttable_row)
+      self._ttables[title]["rows"] = self._ttables[title]["rows"] + ttable_rows
 
   def add_ttable_data(self, template_keyword, title,
                       events_list=None, events_table=None):

diff --git a/stmoab/SummaryDashboard.py b/stmoab/SummaryDashboard.py
@@ -260,7 +260,7 @@ def _template_copy_results_exist(
 
     if data is None or len(data) == 0:
       self._logger.info((
-          "Dashboard: Query '{name}' is still updating and will not be "
+          "Dashboard: Query '{name}' is still updating and will "
           "not be displayed.".format(name=query_title)))
       return False
 

diff --git a/stmoab/dashboard_instances/sample.py b/stmoab/dashboard_instances/sample.py
@@ -174,7 +174,8 @@
     "Pocket Experiment",
     "Sponsored Stories",
     "pref-flip-activity-stream-59-beta-pocket-sponsored-stories-bug-1432989",
-    start_date="2018-01-29"
+    start_date="2018-01-29",
+    end_date="2018-02-13"
 '''
 '''
     "Pocket Experiment",

diff --git a/stmoab/tests/test_statistical_dashboard.py b/stmoab/tests/test_statistical_dashboard.py
@@ -101,7 +101,7 @@ def test_get_ttable_data_for_non_existent_query(self):
     ttable_row = self.dash._get_ttable_data_for_query(
         "beep", "meep", "boop", 5)
 
-    self.assertEqual(ttable_row, {})
+    self.assertEqual(ttable_row, [])
 
   def test_ttable_not_made_for_non_matching_graph(self):
     BAD_ROW = []
@@ -129,14 +129,15 @@ def test_ttable_not_made_for_non_matching_graph(self):
 
   def test_ttable_row_data_is_correct(self):
     EXPECTED_LABEL = "beep"
+    EXPECTED_VARIANT_NAME = "experiment"
     EXPECTED_ROWS = []
     EXPECTED_MEAN_DIFFERENCE = -4
 
     for i in range(12):
       EXPECTED_ROWS.append({
           "date": 123,
           "count": (i % 3) + 1,
-          "type": "experiment"
+          "type": EXPECTED_VARIANT_NAME
       })
       EXPECTED_ROWS.append({
           "date": 123,
@@ -158,13 +159,19 @@ def test_ttable_row_data_is_correct(self):
     ttable_row = self.dash._get_ttable_data_for_query(
         EXPECTED_LABEL, "meep", "count", 5)
 
-    self.assertEqual(len(ttable_row), 8)
-    self.assertEqual(ttable_row["Metric"], EXPECTED_LABEL)
-    self.assertEqual(ttable_row["Alpha Error"], self.dash.ALPHA_ERROR)
-    self.assertTrue(0.5 <= ttable_row["Power"] <= 1)
-    self.assertTrue(0 <= ttable_row["Two-Tailed P-value (ttest)"] <= 0.05)
+    self.assertEqual(len(ttable_row), 1)
+    self.assertEqual(len(ttable_row[0]), 8)
     self.assertEqual(
-        ttable_row["Experiment Mean - Control Mean"], EXPECTED_MEAN_DIFFERENCE)
+        ttable_row[0]["Metric"],
+        "[control vs. {variant}] {title}".format(
+            variant=EXPECTED_VARIANT_NAME,
+            title=EXPECTED_LABEL))
+    self.assertEqual(ttable_row[0]["Alpha Error"], self.dash.ALPHA_ERROR)
+    self.assertTrue(0.5 <= ttable_row[0]["Power"] <= 1)
+    self.assertTrue(0 <= ttable_row[0]["Two-Tailed P-value (ttest)"] <= 0.05)
+    self.assertEqual(
+        ttable_row[0]["Experiment Mean - Control Mean"],
+        EXPECTED_MEAN_DIFFERENCE)
 
   def test_add_ttable_makes_correct_calls(self):
     self.get_calls = 0