[stats] Migrate bayesian engine to gaussian effect model; unlock priors + CUPED

packages/back-end/types/experiment-snapshot.d.ts

@@ -4,7 +4,7 @@ import {
 } from "../src/services/stats";
 import { QueryLanguage } from "./datasource";
 import { MetricInterface, MetricStats } from "./metric";
-import { DifferenceType, StatsEngine } from "./stats";
+import { DifferenceType, RiskType, StatsEngine } from "./stats";
 import { Queries } from "./query";
 import {
   ExperimentReportResultDimension,
@@ -24,6 +24,7 @@ export interface SnapshotMetric {
   ciAdjusted?: [number, number];
   expected?: number;
   risk?: [number, number];
+  riskType?: RiskType;
   stats?: MetricStats;
   pValue?: number;
   pValueAdjusted?: number;

packages/back-end/types/stats.d.ts

@@ -6,6 +6,7 @@ export type PValueCorrection = null | "benjamini-hochberg" | "holm-bonferroni";
 
 export type DifferenceType = "relative" | "absolute" | "scaled";
 
+export type RiskType = "relative" | "absolute";
 interface BaseVariationResponse {
   cr: number;
   value: number;
@@ -25,6 +26,7 @@ interface BaseVariationResponse {
 interface BayesianVariationResponse extends BaseVariationResponse {
   chanceToWin?: number;
   risk?: [number, number];
+  riskType?: RiskType;
 }
 
 interface FrequentistVariationResponse extends BaseVariationResponse {

packages/front-end/services/experiments.ts

@@ -50,8 +50,18 @@ export function getRisk(
   // separate CR because sometimes "baseline" above is the variation
   baselineCR: number
 ): { risk: number; relativeRisk: number; showRisk: boolean } {
-  const risk = stats.risk?.[1] ?? 0;
-  const relativeRisk = baselineCR ? risk / baselineCR : 0;
+  const statsRisk = stats.risk?.[1] ?? 0;
+  let risk: number;
+  let relativeRisk: number;
+  if (stats.riskType === "relative") {
+    risk = statsRisk * baselineCR;
+    relativeRisk = statsRisk;
+  } else {
+    // otherwise it is absolute, including legacy snapshots
+    // that were missing `riskType` field
+    risk = statsRisk;
+    relativeRisk = baselineCR ? statsRisk / baselineCR : 0;
+  }
   const showRisk =
     baseline.cr > 0 &&
     hasEnoughData(baseline, stats, metric, metricDefaults) &&