Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Power analysis follow-up #2446

Merged
merged 4 commits into from
Apr 30, 2024
Merged

Power analysis follow-up #2446

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
94b19af
Add two digits for effect format.
romain-growthbook Apr 29, 2024
615af90
Add tooltips.
romain-growthbook Apr 30, 2024
fcdd819
Proportion/mean
romain-growthbook Apr 30, 2024
8d173ad
Highlight overall week.
romain-growthbook Apr 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
173 changes: 147 additions & 26 deletions packages/front-end/components/PowerCalculation/PowerCalculationContent.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import Link from "next/link";
import { useMemo, useState } from "react";
import clsx from "clsx";
import Tooltip from "@/components/Tooltip/Tooltip";
import { ensureAndReturn } from "@/types/utils";
import { GBHeadingArrowLeft } from "@/components/Icons";
import {
Expand All @@ -11,15 +12,16 @@ import {
} from "./types";
import PowerCalculationStatsEngineModal from "./PowerCalculationStatsEngineModal";

const percentFormatter = (() => {
const formatter = new Intl.NumberFormat(undefined, {
style: "percent",

maximumFractionDigits: 0,
});

return (v: number) => (isNaN(v) ? "N/A" : formatter.format(v));
})();
const percentFormatter = (
v: number,
{ digits }: { digits: number } = { digits: 0 }
) =>
isNaN(v)
? "N/A"
: new Intl.NumberFormat(undefined, {
style: "percent",
maximumFractionDigits: digits,
}).format(v);

const numberFormatter = (() => {
const formatter = Intl.NumberFormat("en-US");
Expand Down Expand Up @@ -153,7 +155,9 @@ const MetricLabel = ({
}) => (
<>
<div className="font-weight-bold">{name}</div>
<div className="small">Effect Size {percentFormatter(effectSize)}</div>
<div className="small">
Effect Size {percentFormatter(effectSize, { digits: 1 })}
</div>
</>
);

Expand Down Expand Up @@ -203,7 +207,7 @@ const SampleSizeAndRuntime = ({
{Object.keys(sampleSizeAndRuntime).map((id) => {
const target = sampleSizeAndRuntime[id];

const { name, effectSize } = ensureAndReturn(
const { name, type, effectSize } = ensureAndReturn(
params.metrics[id]
);

Expand All @@ -217,9 +221,12 @@ const SampleSizeAndRuntime = ({
onClick={() => setSelectedRow(id)}
>
<td>
<MetricLabel name={name} effectSize={effectSize} />
<div className="font-weight-bold">{name}</div>
<div className="small">
{type === "binomial" ? "Proportion" : "Mean"}
</div>
</td>
<td>{percentFormatter(effectSize)}</td>
<td>{percentFormatter(effectSize, { digits: 1 })}</td>
<td>
{target
? `${formatWeeks({
Expand All @@ -241,7 +248,7 @@ const SampleSizeAndRuntime = ({
<p>
Reliably detecting a lift of{" "}
<span className="font-weight-bold">
{percentFormatter(selectedEffectSize)}
{percentFormatter(selectedEffectSize, { digits: 1 })}
</span>{" "}
requires running your experiment for{" "}
{selectedTarget ? (
Expand Down Expand Up @@ -326,17 +333,40 @@ const MinimumDetectableEffect = ({
key={idx}
className={clsx(
results.weekThreshold === idx + 1 &&
"power-analysis-cell-threshold"
"power-analysis-cell-threshold power-analysis-overall-header-threshold"
)}
>
<div className="font-weight-bold">Week {idx + 1}</div>
<span className="small">{numberFormatter(users)} Users</span>
{(() => {
const content = (
<>
<div className="font-weight-bold">Week {idx + 1}</div>
<span className="small">
{numberFormatter(users)} Users
</span>
</>
);

if (results.weekThreshold === idx + 1)
return (
<Tooltip
popperClassName="text-top"
body={`Week ${
idx + 1
} is the first week when all your metrics meet their expected effect size.`}
tipPosition="top"
>
{content}
</Tooltip>
);

return content;
})()}
</th>
))}
</tr>
</thead>
<tbody>
{Object.keys(results.weeks[0]?.metrics).map((id) => (
{Object.keys(results.weeks[0]?.metrics).map((id, pos) => (
<tr key={id}>
<td>
<MetricLabel {...ensureAndReturn(params.metrics[id])} />
Expand All @@ -346,10 +376,44 @@ const MinimumDetectableEffect = ({
key={`${id}-${idx}`}
className={clsx(
ensureAndReturn(metrics[id]).isThreshold &&
"power-analysis-cell-threshold"
"power-analysis-cell-threshold",
results.weekThreshold === idx + 1 &&
"power-analysis-overall-cell-threshold",
Object.keys(results.weeks[0]?.metrics).length == pos + 1 &&
results.weekThreshold === idx + 1 &&
"power-analysis-overall-bottom-threshold"
)}
>
{percentFormatter(ensureAndReturn(metrics[id]).effectSize)}
{(() => {
const content = percentFormatter(
ensureAndReturn(metrics[id]).effectSize,
{
digits: 1,
}
);

if (ensureAndReturn(metrics[id]).isThreshold) {
const { effectSize, name } = ensureAndReturn(
params.metrics[id]
);
return (
<Tooltip
popperClassName="text-top"
body={`Week ${
idx + 1
} is the first week where the minimum detectable effect over time dropped bellow your target effect size of ${percentFormatter(
effectSize,
{ digits: 1 }
)} for your ${name} metric.`}
tipPosition="top"
>
{content}
</Tooltip>
);
}

return content;
})()}
</td>
))}
</tr>
Expand Down Expand Up @@ -387,17 +451,40 @@ const PowerOverTime = ({
key={idx}
className={clsx(
results.weekThreshold === idx + 1 &&
"power-analysis-cell-threshold"
"power-analysis-cell-threshold power-analysis-overall-header-threshold"
)}
>
<div className="font-weight-bold">Week {idx + 1}</div>
<span className="small">{numberFormatter(users)} Users</span>
{(() => {
const content = (
<>
<div className="font-weight-bold">Week {idx + 1}</div>
<span className="small">
{numberFormatter(users)} Users
</span>
</>
);

if (results.weekThreshold === idx + 1)
return (
<Tooltip
popperClassName="text-top"
body={`Week ${
idx + 1
} is the first week when all your metrics meet their expected effect size.`}
tipPosition="top"
>
{content}
</Tooltip>
);

return content;
})()}
</th>
))}
</tr>
</thead>
<tbody>
{Object.keys(results.weeks[0]?.metrics).map((id) => (
{Object.keys(results.weeks[0]?.metrics).map((id, pos) => (
<tr key={id}>
<td>
<MetricLabel {...ensureAndReturn(params.metrics[id])} />
Expand All @@ -407,10 +494,44 @@ const PowerOverTime = ({
key={`${id}-${idx}`}
className={clsx(
ensureAndReturn(metrics[id]).isThreshold &&
"power-analysis-cell-threshold"
"power-analysis-cell-threshold",
results.weekThreshold === idx + 1 &&
"power-analysis-overall-cell-threshold",
Object.keys(results.weeks[0]?.metrics).length == pos + 1 &&
results.weekThreshold === idx + 1 &&
"power-analysis-overall-bottom-threshold"
)}
>
{percentFormatter(ensureAndReturn(metrics[id]).power)}
{(() => {
const content = percentFormatter(
ensureAndReturn(metrics[id]).power
);

if (ensureAndReturn(metrics[id]).isThreshold) {
const { targetPower } = params;
const { effectSize, name } = ensureAndReturn(
params.metrics[id]
);
return (
<Tooltip
popperClassName="text-top"
body={`Week ${
idx + 1
} is the first week with at least ${percentFormatter(
targetPower
)} power to detect an effect size of ${percentFormatter(
effectSize,
{ digits: 1 }
)} for your ${name} metric.`}
tipPosition="top"
>
{content}
</Tooltip>
);
}

return content;
})()}
</td>
))}
</tr>
Expand Down
16 changes: 16 additions & 0 deletions packages/front-end/styles/global.scss
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,22 @@ pre {

.power-analysis-cell-threshold {
background-color: var(--alert-info-background);
cursor: help;
}

.power-analysis-overall-header-threshold {
border-top: 1px solid colors.$highlight-purple !important;
border-left: 1px solid colors.$highlight-purple !important;
border-right: 1px solid colors.$highlight-purple !important;
}

.power-analysis-overall-cell-threshold {
border-left: 1px solid colors.$highlight-purple !important;
border-right: 1px solid colors.$highlight-purple !important;
}

.power-analysis-overall-bottom-threshold {
border-bottom: 1px solid colors.$highlight-purple !important;
}

.w-100px {
Expand Down
Loading