Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix bayes intervals for binry design #45

Open
wants to merge 1 commit into
base: dev
Choose a base branch
from
Open

Fix bayes intervals for binry design #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion ambrosia/tools/_lib/_bin_ci_aide.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ def __helper_calc_empirical_power(conf_interval: types.ManyIntervalType) -> np.n


def __helper_bin_search_for_size(
interval_type: str, confidence_level: float, p_a: float, p_b: float, amount: int, power: float
interval_type: str, confidence_level: float, p_a: float, p_b: float, amount: int, power: float, **kwargs
) -> int:
"""
Make binary search for size to gain given power.
Expand Down Expand Up @@ -81,6 +81,7 @@ def power_helper(trials: int) -> float:
"a_trials": trials,
"b_trials": trials,
"confidence_level": confidence_level,
**kwargs,
}
conf_interval: types.IntervalType = bi.BinomTwoSampleCI.confidence_interval(**binom_kwargs)
return __helper_calc_empirical_power(conf_interval)
Expand Down Expand Up @@ -112,6 +113,7 @@ def __helper_bin_search_for_delta(
amount: int,
power: float,
epsilon: float = 0.0001,
**kwargs,
) -> Optional[float]:
"""
Make binary search for delta to gain given power for
Expand Down Expand Up @@ -154,6 +156,7 @@ def power_helper(delta: float) -> float:
"a_trials": trials,
"b_trials": trials,
"confidence_level": confidence_level,
**kwargs,
}
conf_interval: types.IntervalType = bi.BinomTwoSampleCI.confidence_interval(**binom_kwargs)
return __helper_calc_empirical_power(conf_interval)
Expand Down
17 changes: 14 additions & 3 deletions ambrosia/tools/bin_intervals.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -390,6 +390,7 @@ def get_table_power_on_size_and_conversions(
sample_sizes: Iterable[int] = (100,),
amount: int = 10000,
confidence_level: float = 0.95,
**kwargs,
) -> pd.DataFrame:
"""
Table with power / empirical 1 type error = 1 - coverage, for fixed size and conversions.
Expand Down Expand Up @@ -432,6 +433,7 @@ def get_table_power_on_size_and_conversions(
"a_trials": trials,
"b_trials": trials,
"confidence_level": confidence_level,
**kwargs,
}
conf_interval: types.ManyIntervalType = BinomTwoSampleCI.confidence_interval(**binom_kwargs)
power: np.ndarray = helper_dir.__helper_calc_empirical_power(conf_interval)
Expand All @@ -456,9 +458,9 @@ def get_table_power_on_size_and_delta(
delta_values: Iterable[float] = None,
delta_relative_values: Iterable[float] = None,
interval_type: str = "wald",
# alternative: str = "two-sided",
amount: int = 10000,
as_numeric: bool = False,
**kwargs,
) -> pd.DataFrame:
"""
Table with power / empirical 1 type error = 1 - coverage for fixed size and effect.
Expand Down Expand Up @@ -522,6 +524,7 @@ def get_table_power_on_size_and_delta(
"b_trials": trials,
"confidence_level": 1 - alpha,
# "alternative": alternative,
**kwargs,
}
conf_interval: types.ManyIntervalType = BinomTwoSampleCI.confidence_interval(**binom_kwargs)
power: np.ndarray = helper_dir.__helper_calc_empirical_power(conf_interval)
Expand All @@ -541,6 +544,7 @@ def iterate_for_sample_size(
p_b_values: Iterable[float],
grid_delta: Iterable[float],
amount: int,
**kwargs,
) -> pd.DataFrame:
"""
Iterate over params for different sample size
Expand All @@ -567,6 +571,7 @@ def iterate_for_sample_size(
p_b=p_b,
amount=amount,
power=power,
**kwargs,
)
table.loc[delta, f"({alpha}; {beta})"] = trials
return table
Expand All @@ -580,6 +585,7 @@ def get_table_sample_size_on_effect(
delta_values: Iterable[float] = None,
delta_relative_values: Iterable[float] = None,
amount: int = 10000,
**kwargs,
) -> pd.DataFrame:
"""
Table for sample sizes with given effect and errors.
Expand Down Expand Up @@ -628,7 +634,9 @@ def get_table_sample_size_on_effect(
grid_delta: np.ndarray = [f"{np.round((x - 1) * 100, ROUND_DIGITS_PERCENT)}%" for x in delta_relative_values]
if not np.all((p_b_values >= 0) & (p_b_values <= 1)):
raise ValueError(f"Probability of success in group B must be positive, not {p_b_values}")
table = iterate_for_sample_size(interval_type, first_errors, second_errors, p_a, p_b_values, grid_delta, amount)
table = iterate_for_sample_size(
interval_type, first_errors, second_errors, p_a, p_b_values, grid_delta, amount, **kwargs
)
return table


Expand All @@ -641,6 +649,7 @@ def iterate_for_delta(
amount: int,
delta_type: str,
as_numeric: bool = False,
**kwargs,
) -> pd.DataFrame:
"""
Helps to find effect for different params.
Expand All @@ -662,6 +671,7 @@ def iterate_for_delta(
trials=trials,
amount=amount,
power=power,
**kwargs,
)
if delta is not None and delta_type == "relative":
if as_numeric:
Expand All @@ -681,6 +691,7 @@ def get_table_effect_on_sample_size(
amount: int = 10000,
delta_type: str = "relative",
as_numeric: bool = False,
**kwargs,
) -> pd.DataFrame:
"""
Table for effects with given sample sizes and erros.
Expand Down Expand Up @@ -723,6 +734,6 @@ def get_table_effect_on_sample_size(
if delta_type not in delta_types:
raise ValueError(f"Delta type must be absolute relative, not {delta_type}")
table: pd.DataFrame = iterate_for_delta(
interval_type, first_errors, second_errors, sample_sizes, p_a, amount, delta_type, as_numeric
interval_type, first_errors, second_errors, sample_sizes, p_a, amount, delta_type, as_numeric, **kwargs
)
return table
64 changes: 63 additions & 1 deletion tests/test_designer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,15 @@
import pytest
import yaml

from ambrosia.designer import Designer, design, design_binary, load_from_config
from ambrosia.designer import (
Designer,
design,
design_binary,
design_binary_effect,
design_binary_power,
design_binary_size,
load_from_config,
)

store_path: str = "tests/configs/dumped_designer.yaml"

Expand Down Expand Up @@ -327,3 +335,57 @@ def test_groups_ratio_parameter(to_design, method, effects, sizes, designer_ltv)
)
results_list.append((1.0 + groups_ratio) * res)
assert np.all(results_list[0].values < results_list[1].values < results_list[2].values)


@pytest.mark.smoke
def test_prior_designing_binary():
# Correct prior
n_correct: int = design_binary_size(
0.01,
effects=[1.01],
method="binary",
interval_type="bayes_beta",
n_success_conjugate=1,
n_failure_conjugate=1000002,
).iloc[0, 0]
# Incorrect prior
n_incorrect: int = design_binary_size(
0.99,
effects=[1.01],
methdo="binary",
interval_type="bayes_beta",
n_success_conjugate=1,
n_failure_conjugate=1000002,
).iloc[0, 0]
assert n_correct > n_incorrect

# Test effect
effect_correct = design_binary_effect(
0.1, sizes=[5000], interval_type="bayes_beta", n_success_conjugate=1, n_failure_conjugate=10, as_numeric=True
).iloc[0, 0]
effect_incorrect = design_binary_effect(
0.9, sizes=[5000], interval_type="bayes_beta", n_success_conjugate=1, n_failure_conjugate=10, as_numeric=True
).iloc[0, 0]
assert effect_correct > effect_incorrect

# Test power
power_correct = design_binary_power(
0.1,
sizes=5000,
effects=1.05,
interval_type="bayes_beta",
n_success_conjugate=1,
n_failure_conjugate=10,
as_numeric=True,
).iloc[0, 0]
power_incorrect = design_binary_power(
0.9,
sizes=5000,
effects=1.05,
interval_type="bayes_beta",
n_success_conjugate=1,
n_failure_conjugate=10,
as_numeric=True,
).iloc[0, 0]

assert power_correct < power_incorrect