From 1483053a1aef171b66f2405c588ab1869b13e958 Mon Sep 17 00:00:00 2001 From: Gagan Trivedi Date: Tue, 3 Mar 2026 09:49:50 +0530 Subject: [PATCH 1/4] fix: replace numpy/scipy with pure Python in experiment analytics Removes significant per-worker RSS overhead by replacing numpy and scipy with stdlib math and random modules. The experiment analytics endpoint is demo-only code, so the minor latency trade-off is well worth the memory savings. - Implement G-test via log-likelihood ratio + incomplete gamma function - Use random.betavariate for Bayesian Monte Carlo (10k samples) - Remove numpy and scipy from pyproject.toml and poetry.lock - Convert tests from class-based to function-based --- api/app_analytics/experiments.py | 197 ++++++++---- api/poetry.lock | 186 +---------- api/pyproject.toml | 2 - .../unit/app_analytics/test_experiments.py | 290 +++++++++--------- 4 files changed, 284 insertions(+), 391 deletions(-) diff --git a/api/app_analytics/experiments.py b/api/app_analytics/experiments.py index b4f73cbc0e24..dc5701e955b9 100644 --- a/api/app_analytics/experiments.py +++ b/api/app_analytics/experiments.py @@ -18,9 +18,10 @@ GET /api/v1/environments/{env}/experiments/results/?feature=checkout_button """ +import math +import random from typing import Any -import numpy as np from common.environments.permissions import VIEW_ENVIRONMENT from django.db.models import Count, Q from drf_spectacular.utils import OpenApiParameter, extend_schema @@ -29,7 +30,6 @@ from rest_framework.permissions import IsAuthenticated from rest_framework.request import Request from rest_framework.response import Response -from scipy import stats as scipy_stats # type: ignore[import-untyped] from environments.identities.traits.models import Trait from environments.models import Environment @@ -146,43 +146,34 @@ def _calculate_two_variant_stats( "sample_size_warning": "One or more variants have zero samples", } - # Build contingency table - table = np.array( + # Build contingency table, replacing zeros with 0.5 to avoid log(0) + table = [ [ - [ - results[a]["conversions"], - total_a - results[a]["conversions"], - ], - [ - results[b]["conversions"], - total_b - results[b]["conversions"], - ], - ] - ) + max(results[a]["conversions"], 0.5), + max(total_a - results[a]["conversions"], 0.5), + ], + [ + max(results[b]["conversions"], 0.5), + max(total_b - results[b]["conversions"], 0.5), + ], + ] # G-test (log-likelihood ratio test) - # Replace zeros with small values to avoid division errors - table_safe = np.where(table == 0, 0.5, table) - try: - _, p_value, _, _ = scipy_stats.chi2_contingency( - table_safe, correction=True, lambda_="log-likelihood" - ) - except ValueError: - p_value = 1.0 + p_value = _g_test(table) # Bayesian "Chance to Win" using Monte Carlo simulation - samples = 50000 - samples_a = np.random.beta( - results[a]["conversions"] + 1, - total_a - results[a]["conversions"] + 1, - samples, - ) - samples_b = np.random.beta( - results[b]["conversions"] + 1, - total_b - results[b]["conversions"] + 1, - samples, - ) - chance_a_wins = float((samples_a > samples_b).mean()) + n_samples = 10000 + a_wins = 0 + alpha_a = results[a]["conversions"] + 1 + beta_a = total_a - results[a]["conversions"] + 1 + alpha_b = results[b]["conversions"] + 1 + beta_b = total_b - results[b]["conversions"] + 1 + for _ in range(n_samples): + sa = random.betavariate(alpha_a, beta_a) + sb = random.betavariate(alpha_b, beta_b) + if sa > sb: + a_wins += 1 + chance_a_wins = a_wins / n_samples # Determine winner and calculate lift (winner vs loser) if results[a]["rate"] > results[b]["rate"]: @@ -239,41 +230,33 @@ def _calculate_multi_variant_stats( "sample_size_warning": f"Variant '{v}' has zero samples", } - # Build contingency table for all variants - table = np.array( + # Build contingency table, replacing zeros with 0.5 to avoid log(0) + table = [ [ - [results[v]["conversions"], results[v]["total"] - results[v]["conversions"]] - for v in variants + max(results[v]["conversions"], 0.5), + max(results[v]["total"] - results[v]["conversions"], 0.5), ] - ) + for v in variants + ] # G-test - table_safe = np.where(table == 0, 0.5, table) - try: - _, p_value, _, _ = scipy_stats.chi2_contingency( - table_safe, correction=True, lambda_="log-likelihood" - ) - except ValueError: - p_value = 1.0 + p_value = _g_test(table) - # Bayesian chance to win for each variant - samples = 50000 - variant_samples = {} - for v in variants: - variant_samples[v] = np.random.beta( - results[v]["conversions"] + 1, - results[v]["total"] - results[v]["conversions"] + 1, - samples, - ) - - # Calculate chance each variant is the best - chance_to_win = {} - for v in variants: - wins = np.ones(samples, dtype=bool) - for other in variants: - if other != v: - wins &= variant_samples[v] > variant_samples[other] - chance_to_win[v] = round(float(wins.mean()), 3) + # Bayesian chance to win for each variant via Monte Carlo + n_samples = 10000 + params = [ + (results[v]["conversions"] + 1, results[v]["total"] - results[v]["conversions"] + 1) + for v in variants + ] + win_counts = [0] * len(variants) + for _ in range(n_samples): + draws = [random.betavariate(a, b) for a, b in params] + best_idx = max(range(len(draws)), key=lambda i: draws[i]) + win_counts[best_idx] += 1 + + chance_to_win = { + v: round(win_counts[i] / n_samples, 3) for i, v in enumerate(variants) + } # Find the best performing variant best_variant = max(variants, key=lambda v: results[v]["rate"]) @@ -296,6 +279,92 @@ def _calculate_multi_variant_stats( } +def _g_test(table: list[list[float]]) -> float: + """ + Perform a G-test (log-likelihood ratio test) on a contingency table. + + Returns the p-value under the chi-squared distribution. + """ + rows = len(table) + cols = len(table[0]) + + row_totals = [sum(row) for row in table] + col_totals = [sum(table[r][c] for r in range(rows)) for c in range(cols)] + grand_total = sum(row_totals) + + if grand_total == 0: + return 1.0 + + g = 0.0 + for r in range(rows): + for c in range(cols): + observed = table[r][c] + expected = row_totals[r] * col_totals[c] / grand_total + if observed > 0 and expected > 0: + g += observed * math.log(observed / expected) + g *= 2 + + df = (rows - 1) * (cols - 1) + if df == 0: + return 1.0 + + return _chi2_sf(g, df) + + +def _chi2_sf(x: float, df: int) -> float: + """Survival function (1 - CDF) for the chi-squared distribution.""" + if x <= 0: + return 1.0 + return _upper_gamma_reg(df / 2.0, x / 2.0) + + +def _upper_gamma_reg(a: float, x: float) -> float: + """Regularised upper incomplete gamma function Q(a, x).""" + if x <= 0: + return 1.0 + if x < a + 1: + return 1.0 - _lower_gamma_series(a, x) + return _upper_gamma_cf(a, x) + + +def _lower_gamma_series(a: float, x: float) -> float: + """Regularised lower incomplete gamma P(a, x) via series expansion.""" + ap = a + total = 1.0 / a + term = 1.0 / a + for _ in range(300): + ap += 1 + term *= x / ap + total += term + if abs(term) < abs(total) * 1e-15: + break + return total * math.exp(-x + a * math.log(x) - math.lgamma(a)) + + +def _upper_gamma_cf(a: float, x: float) -> float: + """Regularised upper incomplete gamma Q(a, x) via continued fraction.""" + tiny = 1e-30 + b = x + 1.0 - a + c = 1.0 / tiny + d = 1.0 / b + h = d + for i in range(1, 300): + an = -i * (i - a) + b += 2.0 + d = an * d + b + if abs(d) < tiny: + d = tiny + c = b + an / c + if abs(c) < tiny: + c = tiny + d = 1.0 / d + delta = d * c + h *= delta + if abs(delta - 1.0) < 1e-15: + break + return h * math.exp(-x + a * math.log(x) - math.lgamma(a)) + + @extend_schema( parameters=[ OpenApiParameter( diff --git a/api/poetry.lock b/api/poetry.lock index 8e12abdfd21d..15d608cb26c3 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand. [[package]] name = "annotated-types" @@ -2199,12 +2199,12 @@ files = [ ] [package.dependencies] -google-api-core = {version = ">=1.21.0,<3.dev0", markers = "python_version >= \"3\""} -google-auth = {version = ">=1.16.0,<3.dev0", markers = "python_version >= \"3\""} +google-api-core = {version = ">=1.21.0,<3dev", markers = "python_version >= \"3\""} +google-auth = {version = ">=1.16.0,<3dev", markers = "python_version >= \"3\""} google-auth-httplib2 = ">=0.0.3" -httplib2 = ">=0.15.0,<1.dev0" -six = ">=1.13.0,<2.dev0" -uritemplate = ">=3.0.0,<4.dev0" +httplib2 = ">=0.15.0,<1dev" +six = ">=1.13.0,<2dev" +uritemplate = ">=3.0.0,<4dev" [[package]] name = "google-auth" @@ -2517,7 +2517,7 @@ files = [ ] [package.dependencies] -certifi = ">=14.5.14" +certifi = ">=14.05.14" python-dateutil = ">=2.5.3" reactivex = ">=4.0.4" urllib3 = ">=1.26.0" @@ -2723,7 +2723,7 @@ files = [ [package.dependencies] attrs = ">=22.2.0" -jsonschema-specifications = ">=2023.3.6" +jsonschema-specifications = ">=2023.03.6" referencing = ">=0.28.4" rpds-py = ">=0.7.1" @@ -3100,88 +3100,6 @@ files = [ {file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"}, ] -[[package]] -name = "numpy" -version = "2.4.2" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.11" -groups = ["main"] -files = [ - {file = "numpy-2.4.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e7e88598032542bd49af7c4747541422884219056c268823ef6e5e89851c8825"}, - {file = "numpy-2.4.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7edc794af8b36ca37ef5fcb5e0d128c7e0595c7b96a2318d1badb6fcd8ee86b1"}, - {file = "numpy-2.4.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:6e9f61981ace1360e42737e2bae58b27bf28a1b27e781721047d84bd754d32e7"}, - {file = "numpy-2.4.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:cb7bbb88aa74908950d979eeaa24dbdf1a865e3c7e45ff0121d8f70387b55f73"}, - {file = "numpy-2.4.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4f069069931240b3fc703f1e23df63443dbd6390614c8c44a87d96cd0ec81eb1"}, - {file = "numpy-2.4.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c02ef4401a506fb60b411467ad501e1429a3487abca4664871d9ae0b46c8ba32"}, - {file = "numpy-2.4.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2653de5c24910e49c2b106499803124dde62a5a1fe0eedeaecf4309a5f639390"}, - {file = "numpy-2.4.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1ae241bbfc6ae276f94a170b14785e561cb5e7f626b6688cf076af4110887413"}, - {file = "numpy-2.4.2-cp311-cp311-win32.whl", hash = "sha256:df1b10187212b198dd45fa943d8985a3c8cf854aed4923796e0e019e113a1bda"}, - {file = "numpy-2.4.2-cp311-cp311-win_amd64.whl", hash = "sha256:b9c618d56a29c9cb1c4da979e9899be7578d2e0b3c24d52079c166324c9e8695"}, - {file = "numpy-2.4.2-cp311-cp311-win_arm64.whl", hash = "sha256:47c5a6ed21d9452b10227e5e8a0e1c22979811cad7dcc19d8e3e2fb8fa03f1a3"}, - {file = "numpy-2.4.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:21982668592194c609de53ba4933a7471880ccbaadcc52352694a59ecc860b3a"}, - {file = "numpy-2.4.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40397bda92382fcec844066efb11f13e1c9a3e2a8e8f318fb72ed8b6db9f60f1"}, - {file = "numpy-2.4.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:b3a24467af63c67829bfaa61eecf18d5432d4f11992688537be59ecd6ad32f5e"}, - {file = "numpy-2.4.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:805cc8de9fd6e7a22da5aed858e0ab16be5a4db6c873dde1d7451c541553aa27"}, - {file = "numpy-2.4.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d82351358ffbcdcd7b686b90742a9b86632d6c1c051016484fa0b326a0a1548"}, - {file = "numpy-2.4.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e35d3e0144137d9fdae62912e869136164534d64a169f86438bc9561b6ad49f"}, - {file = "numpy-2.4.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:adb6ed2ad29b9e15321d167d152ee909ec73395901b70936f029c3bc6d7f4460"}, - {file = "numpy-2.4.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8906e71fd8afcb76580404e2a950caef2685df3d2a57fe82a86ac8d33cc007ba"}, - {file = "numpy-2.4.2-cp312-cp312-win32.whl", hash = "sha256:ec055f6dae239a6299cace477b479cca2fc125c5675482daf1dd886933a1076f"}, - {file = "numpy-2.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:209fae046e62d0ce6435fcfe3b1a10537e858249b3d9b05829e2a05218296a85"}, - {file = "numpy-2.4.2-cp312-cp312-win_arm64.whl", hash = "sha256:fbde1b0c6e81d56f5dccd95dd4a711d9b95df1ae4009a60887e56b27e8d903fa"}, - {file = "numpy-2.4.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:25f2059807faea4b077a2b6837391b5d830864b3543627f381821c646f31a63c"}, - {file = "numpy-2.4.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bd3a7a9f5847d2fb8c2c6d1c862fa109c31a9abeca1a3c2bd5a64572955b2979"}, - {file = "numpy-2.4.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:8e4549f8a3c6d13d55041925e912bfd834285ef1dd64d6bc7d542583355e2e98"}, - {file = "numpy-2.4.2-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:aea4f66ff44dfddf8c2cffd66ba6538c5ec67d389285292fe428cb2c738c8aef"}, - {file = "numpy-2.4.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3cd545784805de05aafe1dde61752ea49a359ccba9760c1e5d1c88a93bbf2b7"}, - {file = "numpy-2.4.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d0d9b7c93578baafcbc5f0b83eaf17b79d345c6f36917ba0c67f45226911d499"}, - {file = "numpy-2.4.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f74f0f7779cc7ae07d1810aab8ac6b1464c3eafb9e283a40da7309d5e6e48fbb"}, - {file = "numpy-2.4.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c7ac672d699bf36275c035e16b65539931347d68b70667d28984c9fb34e07fa7"}, - {file = "numpy-2.4.2-cp313-cp313-win32.whl", hash = "sha256:8e9afaeb0beff068b4d9cd20d322ba0ee1cecfb0b08db145e4ab4dd44a6b5110"}, - {file = "numpy-2.4.2-cp313-cp313-win_amd64.whl", hash = "sha256:7df2de1e4fba69a51c06c28f5a3de36731eb9639feb8e1cf7e4a7b0daf4cf622"}, - {file = "numpy-2.4.2-cp313-cp313-win_arm64.whl", hash = "sha256:0fece1d1f0a89c16b03442eae5c56dc0be0c7883b5d388e0c03f53019a4bfd71"}, - {file = "numpy-2.4.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5633c0da313330fd20c484c78cdd3f9b175b55e1a766c4a174230c6b70ad8262"}, - {file = "numpy-2.4.2-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:d9f64d786b3b1dd742c946c42d15b07497ed14af1a1f3ce840cce27daa0ce913"}, - {file = "numpy-2.4.2-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:b21041e8cb6a1eb5312dd1d2f80a94d91efffb7a06b70597d44f1bd2dfc315ab"}, - {file = "numpy-2.4.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:00ab83c56211a1d7c07c25e3217ea6695e50a3e2f255053686b081dc0b091a82"}, - {file = "numpy-2.4.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2fb882da679409066b4603579619341c6d6898fc83a8995199d5249f986e8e8f"}, - {file = "numpy-2.4.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:66cb9422236317f9d44b67b4d18f44efe6e9c7f8794ac0462978513359461554"}, - {file = "numpy-2.4.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:0f01dcf33e73d80bd8dc0f20a71303abbafa26a19e23f6b68d1aa9990af90257"}, - {file = "numpy-2.4.2-cp313-cp313t-win32.whl", hash = "sha256:52b913ec40ff7ae845687b0b34d8d93b60cb66dcee06996dd5c99f2fc9328657"}, - {file = "numpy-2.4.2-cp313-cp313t-win_amd64.whl", hash = "sha256:5eea80d908b2c1f91486eb95b3fb6fab187e569ec9752ab7d9333d2e66bf2d6b"}, - {file = "numpy-2.4.2-cp313-cp313t-win_arm64.whl", hash = "sha256:fd49860271d52127d61197bb50b64f58454e9f578cb4b2c001a6de8b1f50b0b1"}, - {file = "numpy-2.4.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:444be170853f1f9d528428eceb55f12918e4fda5d8805480f36a002f1415e09b"}, - {file = "numpy-2.4.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d1240d50adff70c2a88217698ca844723068533f3f5c5fa6ee2e3220e3bdb000"}, - {file = "numpy-2.4.2-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:7cdde6de52fb6664b00b056341265441192d1291c130e99183ec0d4b110ff8b1"}, - {file = "numpy-2.4.2-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:cda077c2e5b780200b6b3e09d0b42205a3d1c68f30c6dceb90401c13bff8fe74"}, - {file = "numpy-2.4.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d30291931c915b2ab5717c2974bb95ee891a1cf22ebc16a8006bd59cd210d40a"}, - {file = "numpy-2.4.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bba37bc29d4d85761deed3954a1bc62be7cf462b9510b51d367b769a8c8df325"}, - {file = "numpy-2.4.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b2f0073ed0868db1dcd86e052d37279eef185b9c8db5bf61f30f46adac63c909"}, - {file = "numpy-2.4.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7f54844851cdb630ceb623dcec4db3240d1ac13d4990532446761baede94996a"}, - {file = "numpy-2.4.2-cp314-cp314-win32.whl", hash = "sha256:12e26134a0331d8dbd9351620f037ec470b7c75929cb8a1537f6bfe411152a1a"}, - {file = "numpy-2.4.2-cp314-cp314-win_amd64.whl", hash = "sha256:068cdb2d0d644cdb45670810894f6a0600797a69c05f1ac478e8d31670b8ee75"}, - {file = "numpy-2.4.2-cp314-cp314-win_arm64.whl", hash = "sha256:6ed0be1ee58eef41231a5c943d7d1375f093142702d5723ca2eb07db9b934b05"}, - {file = "numpy-2.4.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:98f16a80e917003a12c0580f97b5f875853ebc33e2eaa4bccfc8201ac6869308"}, - {file = "numpy-2.4.2-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:20abd069b9cda45874498b245c8015b18ace6de8546bf50dfa8cea1696ed06ef"}, - {file = "numpy-2.4.2-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:e98c97502435b53741540a5717a6749ac2ada901056c7db951d33e11c885cc7d"}, - {file = "numpy-2.4.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:da6cad4e82cb893db4b69105c604d805e0c3ce11501a55b5e9f9083b47d2ffe8"}, - {file = "numpy-2.4.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e4424677ce4b47fe73c8b5556d876571f7c6945d264201180db2dc34f676ab5"}, - {file = "numpy-2.4.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2b8f157c8a6f20eb657e240f8985cc135598b2b46985c5bccbde7616dc9c6b1e"}, - {file = "numpy-2.4.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5daf6f3914a733336dab21a05cdec343144600e964d2fcdabaac0c0269874b2a"}, - {file = "numpy-2.4.2-cp314-cp314t-win32.whl", hash = "sha256:8c50dd1fc8826f5b26a5ee4d77ca55d88a895f4e4819c7ecc2a9f5905047a443"}, - {file = "numpy-2.4.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fcf92bee92742edd401ba41135185866f7026c502617f422eb432cfeca4fe236"}, - {file = "numpy-2.4.2-cp314-cp314t-win_arm64.whl", hash = "sha256:1f92f53998a17265194018d1cc321b2e96e900ca52d54c7c77837b71b9465181"}, - {file = "numpy-2.4.2-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:89f7268c009bc492f506abd6f5265defa7cb3f7487dc21d357c3d290add45082"}, - {file = "numpy-2.4.2-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:e6dee3bb76aa4009d5a912180bf5b2de012532998d094acee25d9cb8dee3e44a"}, - {file = "numpy-2.4.2-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:cd2bd2bbed13e213d6b55dc1d035a4f91748a7d3edc9480c13898b0353708920"}, - {file = "numpy-2.4.2-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:cf28c0c1d4c4bf00f509fa7eb02c58d7caf221b50b467bcb0d9bbf1584d5c821"}, - {file = "numpy-2.4.2-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e04ae107ac591763a47398bb45b568fc38f02dbc4aa44c063f67a131f99346cb"}, - {file = "numpy-2.4.2-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:602f65afdef699cda27ec0b9224ae5dc43e328f4c24c689deaf77133dbee74d0"}, - {file = "numpy-2.4.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:be71bf1edb48ebbbf7f6337b5bfd2f895d1902f6335a5830b20141fc126ffba0"}, - {file = "numpy-2.4.2.tar.gz", hash = "sha256:659a6107e31a83c4e33f763942275fd278b21d095094044eb35569e86a21ddae"}, -] - [[package]] name = "oauth2client" version = "4.1.3" @@ -3838,7 +3756,7 @@ files = [ ] [package.dependencies] -astroid = ">=2.14.2,<=2.16.0.dev0" +astroid = ">=2.14.2,<=2.16.0-dev0" colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} dill = {version = ">=0.3.6", markers = "python_version >= \"3.11\""} isort = ">=4.2.5,<6" @@ -4804,89 +4722,10 @@ files = [ ] [package.dependencies] -botocore = ">=1.33.2,<2.0a0" - -[package.extras] -crt = ["botocore[crt] (>=1.33.2,<2.0a0)"] - -[[package]] -name = "scipy" -version = "1.17.0" -description = "Fundamental algorithms for scientific computing in Python" -optional = false -python-versions = ">=3.11" -groups = ["main"] -files = [ - {file = "scipy-1.17.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:2abd71643797bd8a106dff97894ff7869eeeb0af0f7a5ce02e4227c6a2e9d6fd"}, - {file = "scipy-1.17.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:ef28d815f4d2686503e5f4f00edc387ae58dfd7a2f42e348bb53359538f01558"}, - {file = "scipy-1.17.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:272a9f16d6bb4667e8b50d25d71eddcc2158a214df1b566319298de0939d2ab7"}, - {file = "scipy-1.17.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:7204fddcbec2fe6598f1c5fdf027e9f259106d05202a959a9f1aecf036adc9f6"}, - {file = "scipy-1.17.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fc02c37a5639ee67d8fb646ffded6d793c06c5622d36b35cfa8fe5ececb8f042"}, - {file = "scipy-1.17.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dac97a27520d66c12a34fd90a4fe65f43766c18c0d6e1c0a80f114d2260080e4"}, - {file = "scipy-1.17.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ebb7446a39b3ae0fe8f416a9a3fdc6fba3f11c634f680f16a239c5187bc487c0"}, - {file = "scipy-1.17.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:474da16199f6af66601a01546144922ce402cb17362e07d82f5a6cf8f963e449"}, - {file = "scipy-1.17.0-cp311-cp311-win_amd64.whl", hash = "sha256:255c0da161bd7b32a6c898e7891509e8a9289f0b1c6c7d96142ee0d2b114c2ea"}, - {file = "scipy-1.17.0-cp311-cp311-win_arm64.whl", hash = "sha256:85b0ac3ad17fa3be50abd7e69d583d98792d7edc08367e01445a1e2076005379"}, - {file = "scipy-1.17.0-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:0d5018a57c24cb1dd828bcf51d7b10e65986d549f52ef5adb6b4d1ded3e32a57"}, - {file = "scipy-1.17.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:88c22af9e5d5a4f9e027e26772cc7b5922fab8bcc839edb3ae33de404feebd9e"}, - {file = "scipy-1.17.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:f3cd947f20fe17013d401b64e857c6b2da83cae567adbb75b9dcba865abc66d8"}, - {file = "scipy-1.17.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:e8c0b331c2c1f531eb51f1b4fc9ba709521a712cce58f1aa627bc007421a5306"}, - {file = "scipy-1.17.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5194c445d0a1c7a6c1a4a4681b6b7c71baad98ff66d96b949097e7513c9d6742"}, - {file = "scipy-1.17.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9eeb9b5f5997f75507814ed9d298ab23f62cf79f5a3ef90031b1ee2506abdb5b"}, - {file = "scipy-1.17.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:40052543f7bbe921df4408f46003d6f01c6af109b9e2c8a66dd1cf6cf57f7d5d"}, - {file = "scipy-1.17.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0cf46c8013fec9d3694dc572f0b54100c28405d55d3e2cb15e2895b25057996e"}, - {file = "scipy-1.17.0-cp312-cp312-win_amd64.whl", hash = "sha256:0937a0b0d8d593a198cededd4c439a0ea216a3f36653901ea1f3e4be949056f8"}, - {file = "scipy-1.17.0-cp312-cp312-win_arm64.whl", hash = "sha256:f603d8a5518c7426414d1d8f82e253e454471de682ce5e39c29adb0df1efb86b"}, - {file = "scipy-1.17.0-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:65ec32f3d32dfc48c72df4291345dae4f048749bc8d5203ee0a3f347f96c5ce6"}, - {file = "scipy-1.17.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:1f9586a58039d7229ce77b52f8472c972448cded5736eaf102d5658bbac4c269"}, - {file = "scipy-1.17.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:9fad7d3578c877d606b1150135c2639e9de9cecd3705caa37b66862977cc3e72"}, - {file = "scipy-1.17.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:423ca1f6584fc03936972b5f7c06961670dbba9f234e71676a7c7ccf938a0d61"}, - {file = "scipy-1.17.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe508b5690e9eaaa9467fc047f833af58f1152ae51a0d0aed67aa5801f4dd7d6"}, - {file = "scipy-1.17.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6680f2dfd4f6182e7d6db161344537da644d1cf85cf293f015c60a17ecf08752"}, - {file = "scipy-1.17.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:eec3842ec9ac9de5917899b277428886042a93db0b227ebbe3a333b64ec7643d"}, - {file = "scipy-1.17.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d7425fcafbc09a03731e1bc05581f5fad988e48c6a861f441b7ab729a49a55ea"}, - {file = "scipy-1.17.0-cp313-cp313-win_amd64.whl", hash = "sha256:87b411e42b425b84777718cc41516b8a7e0795abfa8e8e1d573bf0ef014f0812"}, - {file = "scipy-1.17.0-cp313-cp313-win_arm64.whl", hash = "sha256:357ca001c6e37601066092e7c89cca2f1ce74e2a520ca78d063a6d2201101df2"}, - {file = "scipy-1.17.0-cp313-cp313t-macosx_10_14_x86_64.whl", hash = "sha256:ec0827aa4d36cb79ff1b81de898e948a51ac0b9b1c43e4a372c0508c38c0f9a3"}, - {file = "scipy-1.17.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:819fc26862b4b3c73a60d486dbb919202f3d6d98c87cf20c223511429f2d1a97"}, - {file = "scipy-1.17.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:363ad4ae2853d88ebcde3ae6ec46ccca903ea9835ee8ba543f12f575e7b07e4e"}, - {file = "scipy-1.17.0-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:979c3a0ff8e5ba254d45d59ebd38cde48fce4f10b5125c680c7a4bfe177aab07"}, - {file = "scipy-1.17.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:130d12926ae34399d157de777472bf82e9061c60cc081372b3118edacafe1d00"}, - {file = "scipy-1.17.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e886000eb4919eae3a44f035e63f0fd8b651234117e8f6f29bad1cd26e7bc45"}, - {file = "scipy-1.17.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:13c4096ac6bc31d706018f06a49abe0485f96499deb82066b94d19b02f664209"}, - {file = "scipy-1.17.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cacbaddd91fcffde703934897c5cd2c7cb0371fac195d383f4e1f1c5d3f3bd04"}, - {file = "scipy-1.17.0-cp313-cp313t-win_amd64.whl", hash = "sha256:edce1a1cf66298cccdc48a1bdf8fb10a3bf58e8b58d6c3883dd1530e103f87c0"}, - {file = "scipy-1.17.0-cp313-cp313t-win_arm64.whl", hash = "sha256:30509da9dbec1c2ed8f168b8d8aa853bc6723fede1dbc23c7d43a56f5ab72a67"}, - {file = "scipy-1.17.0-cp314-cp314-macosx_10_14_x86_64.whl", hash = "sha256:c17514d11b78be8f7e6331b983a65a7f5ca1fd037b95e27b280921fe5606286a"}, - {file = "scipy-1.17.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:4e00562e519c09da34c31685f6acc3aa384d4d50604db0f245c14e1b4488bfa2"}, - {file = "scipy-1.17.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:f7df7941d71314e60a481e02d5ebcb3f0185b8d799c70d03d8258f6c80f3d467"}, - {file = "scipy-1.17.0-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:aabf057c632798832f071a8dde013c2e26284043934f53b00489f1773b33527e"}, - {file = "scipy-1.17.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a38c3337e00be6fd8a95b4ed66b5d988bac4ec888fd922c2ea9fe5fb1603dd67"}, - {file = "scipy-1.17.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00fb5f8ec8398ad90215008d8b6009c9db9fa924fd4c7d6be307c6f945f9cd73"}, - {file = "scipy-1.17.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f2a4942b0f5f7c23c7cd641a0ca1955e2ae83dedcff537e3a0259096635e186b"}, - {file = "scipy-1.17.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:dbf133ced83889583156566d2bdf7a07ff89228fe0c0cb727f777de92092ec6b"}, - {file = "scipy-1.17.0-cp314-cp314-win_amd64.whl", hash = "sha256:3625c631a7acd7cfd929e4e31d2582cf00f42fcf06011f59281271746d77e061"}, - {file = "scipy-1.17.0-cp314-cp314-win_arm64.whl", hash = "sha256:9244608d27eafe02b20558523ba57f15c689357c85bdcfe920b1828750aa26eb"}, - {file = "scipy-1.17.0-cp314-cp314t-macosx_10_14_x86_64.whl", hash = "sha256:2b531f57e09c946f56ad0b4a3b2abee778789097871fc541e267d2eca081cff1"}, - {file = "scipy-1.17.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:13e861634a2c480bd237deb69333ac79ea1941b94568d4b0efa5db5e263d4fd1"}, - {file = "scipy-1.17.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:eb2651271135154aa24f6481cbae5cc8af1f0dd46e6533fb7b56aa9727b6a232"}, - {file = "scipy-1.17.0-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:c5e8647f60679790c2f5c76be17e2e9247dc6b98ad0d3b065861e082c56e078d"}, - {file = "scipy-1.17.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5fb10d17e649e1446410895639f3385fd2bf4c3c7dfc9bea937bddcbc3d7b9ba"}, - {file = "scipy-1.17.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8547e7c57f932e7354a2319fab613981cde910631979f74c9b542bb167a8b9db"}, - {file = "scipy-1.17.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:33af70d040e8af9d5e7a38b5ed3b772adddd281e3062ff23fec49e49681c38cf"}, - {file = "scipy-1.17.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f9eb55bb97d00f8b7ab95cb64f873eb0bf54d9446264d9f3609130381233483f"}, - {file = "scipy-1.17.0-cp314-cp314t-win_amd64.whl", hash = "sha256:1ff269abf702f6c7e67a4b7aad981d42871a11b9dd83c58d2d2ea624efbd1088"}, - {file = "scipy-1.17.0-cp314-cp314t-win_arm64.whl", hash = "sha256:031121914e295d9791319a1875444d55079885bbae5bdc9c5e0f2ee5f09d34ff"}, - {file = "scipy-1.17.0.tar.gz", hash = "sha256:2591060c8e648d8b96439e111ac41fd8342fdeff1876be2e19dea3fe8930454e"}, -] - -[package.dependencies] -numpy = ">=1.26.4,<2.7" +botocore = ">=1.33.2,<2.0a.0" [package.extras] -dev = ["click (<8.3.0)", "cython-lint (>=0.12.2)", "mypy (==1.10.0)", "pycodestyle", "ruff (>=0.12.0)", "spin", "types-psutil", "typing_extensions"] -doc = ["intersphinx_registry", "jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.19.1)", "jupytext", "linkify-it-py", "matplotlib (>=3.5)", "myst-nb (>=1.2.0)", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0,<8.2.0)", "sphinx-copybutton", "sphinx-design (>=0.4.0)", "tabulate"] -test = ["Cython", "array-api-strict (>=2.3.1)", "asv", "gmpy2", "hypothesis (>=6.30)", "meson", "mpmath", "ninja ; sys_platform != \"emscripten\"", "pooch", "pytest (>=8.0.0)", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] +crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] [[package]] name = "segment-analytics-python" @@ -5527,7 +5366,6 @@ files = [ {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, ] -markers = {auth-controller = "sys_platform == \"win32\"", dev = "sys_platform == \"win32\"", ldap = "sys_platform == \"win32\"", workflows = "sys_platform == \"win32\""} [[package]] name = "uritemplate" @@ -5785,4 +5623,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">3.11,<3.14" -content-hash = "76867d44b343c32c8d09f39c5bfcd98a796e28d5e5ef0bfac073a8b1d140fe8e" +content-hash = "f95f290f1c0ede5b477a2a2eb7b06cd450d349b08f89a9ba58184aa981d8be75" diff --git a/api/pyproject.toml b/api/pyproject.toml index 7c4231815ea2..cff57de11374 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -174,8 +174,6 @@ djangorestframework-simplejwt = "^5.5.1" structlog = "^24.4.0" prometheus-client = "^0.21.1" django_cockroachdb = "~4.2" -numpy = "^2.4.2" -scipy = "^1.17.0" [tool.poetry.group.auth-controller] optional = true diff --git a/api/tests/unit/app_analytics/test_experiments.py b/api/tests/unit/app_analytics/test_experiments.py index c06905026479..ffd34549c99b 100644 --- a/api/tests/unit/app_analytics/test_experiments.py +++ b/api/tests/unit/app_analytics/test_experiments.py @@ -1,5 +1,5 @@ -import numpy as np -import pytest +import random + from rest_framework import status from rest_framework.test import APIClient @@ -9,155 +9,143 @@ from environments.models import Environment -class TestCalculateStatistics: - def test_calculate_statistics__two_variants__returns_correct_stats( - self, - ) -> None: - # Given - np.random.seed(42) # For reproducible Bayesian results - results = { - "blue": {"total": 1000, "conversions": 100, "rate": 0.10}, - "green": {"total": 1000, "conversions": 150, "rate": 0.15}, - } - - # When - stats = calculate_statistics(results) - - # Then - assert stats["significant"] is True - assert stats["p_value"] < 0.05 - assert stats["winner"] == "green" - assert stats["chance_to_win"]["green"] > 0.9 - assert "+50" in stats["lift"] or "+49" in stats["lift"] - - def test_calculate_statistics__not_significant__returns_no_winner( - self, - ) -> None: - # Given - np.random.seed(42) - results = { - "blue": {"total": 100, "conversions": 10, "rate": 0.10}, - "green": {"total": 100, "conversions": 11, "rate": 0.11}, - } - - # When - stats = calculate_statistics(results) - - # Then - assert stats["significant"] is False - assert stats["winner"] is None - assert "Keep collecting data" in stats["recommendation"] - - def test_calculate_statistics__single_variant__returns_error_message( - self, - ) -> None: - # Given - results = { - "blue": {"total": 1000, "conversions": 100, "rate": 0.10}, - } - - # When - stats = calculate_statistics(results) - - # Then - assert "Need at least 2 variants" in stats["recommendation"] - - def test_calculate_statistics__three_variants__returns_stats_for_all( - self, - ) -> None: - # Given - np.random.seed(42) - results = { - "blue": {"total": 1000, "conversions": 100, "rate": 0.10}, - "green": {"total": 1000, "conversions": 150, "rate": 0.15}, - "red": {"total": 1000, "conversions": 120, "rate": 0.12}, - } - - # When - stats = calculate_statistics(results) - - # Then - assert "blue" in stats["chance_to_win"] - assert "green" in stats["chance_to_win"] - assert "red" in stats["chance_to_win"] - assert stats["chance_to_win"]["green"] > stats["chance_to_win"]["blue"] - - -@pytest.mark.django_db -class TestGetExperimentResultsView: - def test_get_experiment_results__with_data__returns_results( - self, - environment: Environment, - admin_client_original: APIClient, - ) -> None: - # Given - Create identities with experiment traits - feature_name = "checkout_button" - - for i in range(50): - identity = Identity.objects.create( - identifier=f"user_{i}", - environment=environment, - ) - # Variant assignment - variant = "blue" if i < 25 else "green" +def test_calculate_statistics__two_variants__returns_correct_stats() -> None: + # Given + random.seed(42) + results = { + "blue": {"total": 1000, "conversions": 100, "rate": 0.10}, + "green": {"total": 1000, "conversions": 150, "rate": 0.15}, + } + + # When + stats = calculate_statistics(results) + + # Then + assert stats["significant"] is True + assert stats["p_value"] < 0.05 + assert stats["winner"] == "green" + assert stats["chance_to_win"]["green"] > 0.9 + assert "+50" in stats["lift"] or "+49" in stats["lift"] + + +def test_calculate_statistics__not_significant__returns_no_winner() -> None: + # Given + random.seed(42) + results = { + "blue": {"total": 100, "conversions": 10, "rate": 0.10}, + "green": {"total": 100, "conversions": 11, "rate": 0.11}, + } + + # When + stats = calculate_statistics(results) + + # Then + assert stats["significant"] is False + assert stats["winner"] is None + assert "Keep collecting data" in stats["recommendation"] + + +def test_calculate_statistics__single_variant__returns_error_message() -> None: + # Given + results = { + "blue": {"total": 1000, "conversions": 100, "rate": 0.10}, + } + + # When + stats = calculate_statistics(results) + + # Then + assert "Need at least 2 variants" in stats["recommendation"] + + +def test_calculate_statistics__three_variants__returns_stats_for_all() -> None: + # Given + random.seed(42) + results = { + "blue": {"total": 1000, "conversions": 100, "rate": 0.10}, + "green": {"total": 1000, "conversions": 150, "rate": 0.15}, + "red": {"total": 1000, "conversions": 120, "rate": 0.12}, + } + + # When + stats = calculate_statistics(results) + + # Then + assert "blue" in stats["chance_to_win"] + assert "green" in stats["chance_to_win"] + assert "red" in stats["chance_to_win"] + assert stats["chance_to_win"]["green"] > stats["chance_to_win"]["blue"] + + +def test_get_experiment_results__with_data__returns_results( + environment: Environment, + admin_client_original: APIClient, +) -> None: + # Given + feature_name = "checkout_button" + + for i in range(50): + identity = Identity.objects.create( + identifier=f"user_{i}", + environment=environment, + ) + variant = "blue" if i < 25 else "green" + Trait.objects.create( + identity=identity, + trait_key=f"exp_{feature_name}_variant", + string_value=variant, + value_type="unicode", + ) + if (variant == "blue" and i % 5 == 0) or ( + variant == "green" and i % 3 == 0 + ): Trait.objects.create( identity=identity, - trait_key=f"exp_{feature_name}_variant", - string_value=variant, - value_type="unicode", + trait_key=f"exp_{feature_name}_converted", + boolean_value=True, + value_type="bool", ) - # Some conversions (more for green) - if (variant == "blue" and i % 5 == 0) or ( - variant == "green" and i % 3 == 0 - ): - Trait.objects.create( - identity=identity, - trait_key=f"exp_{feature_name}_converted", - boolean_value=True, - value_type="bool", - ) - - url = f"/api/v1/environments/{environment.api_key}/experiments/results/" - - # When - response = admin_client_original.get(url, {"feature": feature_name}) - - # Then - assert response.status_code == status.HTTP_200_OK - data = response.json() - assert data["feature"] == feature_name - assert "event_type" not in data # Simplified - just converted or not - assert len(data["variants"]) == 2 - assert "statistics" in data - assert "p_value" in data["statistics"] - assert "chance_to_win" in data["statistics"] - - def test_get_experiment_results__no_data__returns_empty_results( - self, - environment: Environment, - admin_client_original: APIClient, - ) -> None: - # Given - url = f"/api/v1/environments/{environment.api_key}/experiments/results/" - - # When - response = admin_client_original.get(url, {"feature": "nonexistent_feature"}) - - # Then - assert response.status_code == status.HTTP_200_OK - data = response.json() - assert data["variants"] == [] - - def test_get_experiment_results__missing_feature_param__returns_400( - self, - environment: Environment, - admin_client_original: APIClient, - ) -> None: - # Given - url = f"/api/v1/environments/{environment.api_key}/experiments/results/" - - # When - response = admin_client_original.get(url) - - # Then - assert response.status_code == status.HTTP_400_BAD_REQUEST + + url = f"/api/v1/environments/{environment.api_key}/experiments/results/" + + # When + response = admin_client_original.get(url, {"feature": feature_name}) + + # Then + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert data["feature"] == feature_name + assert len(data["variants"]) == 2 + assert "statistics" in data + assert "p_value" in data["statistics"] + assert "chance_to_win" in data["statistics"] + + +def test_get_experiment_results__no_data__returns_empty_results( + environment: Environment, + admin_client_original: APIClient, +) -> None: + # Given + url = f"/api/v1/environments/{environment.api_key}/experiments/results/" + + # When + response = admin_client_original.get(url, {"feature": "nonexistent_feature"}) + + # Then + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert data["variants"] == [] + + +def test_get_experiment_results__missing_feature_param__returns_400( + environment: Environment, + admin_client_original: APIClient, +) -> None: + # Given + url = f"/api/v1/environments/{environment.api_key}/experiments/results/" + + # When + response = admin_client_original.get(url) + + # Then + assert response.status_code == status.HTTP_400_BAD_REQUEST From a22589b843edd93d13c28747f37493ed1a72d7ce Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Mar 2026 04:23:15 +0000 Subject: [PATCH 2/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- api/app_analytics/experiments.py | 5 ++++- api/tests/unit/app_analytics/test_experiments.py | 4 +--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/api/app_analytics/experiments.py b/api/app_analytics/experiments.py index dc5701e955b9..86b5dd52c4b9 100644 --- a/api/app_analytics/experiments.py +++ b/api/app_analytics/experiments.py @@ -245,7 +245,10 @@ def _calculate_multi_variant_stats( # Bayesian chance to win for each variant via Monte Carlo n_samples = 10000 params = [ - (results[v]["conversions"] + 1, results[v]["total"] - results[v]["conversions"] + 1) + ( + results[v]["conversions"] + 1, + results[v]["total"] - results[v]["conversions"] + 1, + ) for v in variants ] win_counts = [0] * len(variants) diff --git a/api/tests/unit/app_analytics/test_experiments.py b/api/tests/unit/app_analytics/test_experiments.py index ffd34549c99b..f0401fbfd0da 100644 --- a/api/tests/unit/app_analytics/test_experiments.py +++ b/api/tests/unit/app_analytics/test_experiments.py @@ -96,9 +96,7 @@ def test_get_experiment_results__with_data__returns_results( string_value=variant, value_type="unicode", ) - if (variant == "blue" and i % 5 == 0) or ( - variant == "green" and i % 3 == 0 - ): + if (variant == "blue" and i % 5 == 0) or (variant == "green" and i % 3 == 0): Trait.objects.create( identity=identity, trait_key=f"exp_{feature_name}_converted", From e9012aa943d5e7d91182587229734baf53b29fd2 Mon Sep 17 00:00:00 2001 From: Gagan Trivedi Date: Tue, 3 Mar 2026 10:30:53 +0530 Subject: [PATCH 3/4] test: add edge-case coverage for experiment statistics Cover all new code paths through the public calculate_statistics API and mark unreachable numerical guards with pragma: no cover. --- api/app_analytics/experiments.py | 10 +- .../unit/app_analytics/test_experiments.py | 124 ++++++++++++++++++ 2 files changed, 129 insertions(+), 5 deletions(-) diff --git a/api/app_analytics/experiments.py b/api/app_analytics/experiments.py index 86b5dd52c4b9..dd129b348b15 100644 --- a/api/app_analytics/experiments.py +++ b/api/app_analytics/experiments.py @@ -295,7 +295,7 @@ def _g_test(table: list[list[float]]) -> float: col_totals = [sum(table[r][c] for r in range(rows)) for c in range(cols)] grand_total = sum(row_totals) - if grand_total == 0: + if grand_total == 0: # pragma: no cover return 1.0 g = 0.0 @@ -308,7 +308,7 @@ def _g_test(table: list[list[float]]) -> float: g *= 2 df = (rows - 1) * (cols - 1) - if df == 0: + if df == 0: # pragma: no cover return 1.0 return _chi2_sf(g, df) @@ -323,7 +323,7 @@ def _chi2_sf(x: float, df: int) -> float: def _upper_gamma_reg(a: float, x: float) -> float: """Regularised upper incomplete gamma function Q(a, x).""" - if x <= 0: + if x <= 0: # pragma: no cover return 1.0 if x < a + 1: return 1.0 - _lower_gamma_series(a, x) @@ -355,10 +355,10 @@ def _upper_gamma_cf(a: float, x: float) -> float: an = -i * (i - a) b += 2.0 d = an * d + b - if abs(d) < tiny: + if abs(d) < tiny: # pragma: no cover d = tiny c = b + an / c - if abs(c) < tiny: + if abs(c) < tiny: # pragma: no cover c = tiny d = 1.0 / d delta = d * c diff --git a/api/tests/unit/app_analytics/test_experiments.py b/api/tests/unit/app_analytics/test_experiments.py index f0401fbfd0da..5eb0b2aa0cda 100644 --- a/api/tests/unit/app_analytics/test_experiments.py +++ b/api/tests/unit/app_analytics/test_experiments.py @@ -45,6 +45,130 @@ def test_calculate_statistics__not_significant__returns_no_winner() -> None: assert "Keep collecting data" in stats["recommendation"] +def test_calculate_statistics__no_variants__returns_no_data_message() -> None: + # Given + results: dict[str, dict[str, object]] = {} + + # When + stats = calculate_statistics(results) + + # Then + assert stats["significant"] is False + assert stats["p_value"] == 1.0 + assert stats["recommendation"] == "No experiment data found" + + +def test_calculate_statistics__zero_conversions__returns_not_significant() -> None: + # Given + random.seed(42) + results = { + "blue": {"total": 500, "conversions": 0, "rate": 0.0}, + "green": {"total": 500, "conversions": 0, "rate": 0.0}, + } + + # When + stats = calculate_statistics(results) + + # Then + assert stats["significant"] is False + assert stats["p_value"] == 1.0 + + +def test_calculate_statistics__equal_rates__returns_not_significant() -> None: + # Given + random.seed(42) + results = { + "blue": {"total": 500, "conversions": 50, "rate": 0.10}, + "green": {"total": 500, "conversions": 50, "rate": 0.10}, + } + + # When + stats = calculate_statistics(results) + + # Then + assert stats["significant"] is False + assert stats["winner"] is None + + +def test_calculate_statistics__two_variants_first_wins__returns_correct_winner() -> None: + # Given + random.seed(42) + results = { + "blue": {"total": 1000, "conversions": 150, "rate": 0.15}, + "green": {"total": 1000, "conversions": 100, "rate": 0.10}, + } + + # When + stats = calculate_statistics(results) + + # Then + assert stats["winner"] == "blue" + + +def test_calculate_statistics__moderate_sample__returns_sample_warning() -> None: + # Given + random.seed(42) + results = { + "blue": {"total": 50, "conversions": 5, "rate": 0.10}, + "green": {"total": 50, "conversions": 15, "rate": 0.30}, + } + + # When + stats = calculate_statistics(results) + + # Then + assert stats["sample_size_warning"] is not None + assert "modest" in stats["sample_size_warning"] + + +def test_calculate_statistics__zero_total_variant__returns_insufficient_data() -> None: + # Given + results = { + "blue": {"total": 100, "conversions": 10, "rate": 0.10}, + "green": {"total": 0, "conversions": 0, "rate": 0.0}, + } + + # When + stats = calculate_statistics(results) + + # Then + assert stats["significant"] is False + assert "Insufficient data" in stats["recommendation"] + + +def test_calculate_statistics__multi_variant_zero_total__returns_insufficient_data() -> None: + # Given + results = { + "blue": {"total": 100, "conversions": 10, "rate": 0.10}, + "green": {"total": 100, "conversions": 15, "rate": 0.15}, + "red": {"total": 0, "conversions": 0, "rate": 0.0}, + } + + # When + stats = calculate_statistics(results) + + # Then + assert stats["significant"] is False + assert "Insufficient data" in stats["recommendation"] + + +def test_calculate_statistics__small_sample__uses_series_expansion() -> None: + # Given - small G-statistic triggers series path (x < a+1) + random.seed(42) + results = { + "blue": {"total": 20, "conversions": 2, "rate": 0.10}, + "green": {"total": 20, "conversions": 3, "rate": 0.15}, + } + + # When + stats = calculate_statistics(results) + + # Then + assert stats["significant"] is False + assert 0.0 <= stats["p_value"] <= 1.0 + assert stats["sample_size_warning"] is not None + + def test_calculate_statistics__single_variant__returns_error_message() -> None: # Given results = { From f302c67329d21680ecfba104137a5df6c0312f86 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Mar 2026 05:01:36 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- api/tests/unit/app_analytics/test_experiments.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/api/tests/unit/app_analytics/test_experiments.py b/api/tests/unit/app_analytics/test_experiments.py index 5eb0b2aa0cda..f4413c4eb4fd 100644 --- a/api/tests/unit/app_analytics/test_experiments.py +++ b/api/tests/unit/app_analytics/test_experiments.py @@ -90,7 +90,9 @@ def test_calculate_statistics__equal_rates__returns_not_significant() -> None: assert stats["winner"] is None -def test_calculate_statistics__two_variants_first_wins__returns_correct_winner() -> None: +def test_calculate_statistics__two_variants_first_wins__returns_correct_winner() -> ( + None +): # Given random.seed(42) results = { @@ -136,7 +138,9 @@ def test_calculate_statistics__zero_total_variant__returns_insufficient_data() - assert "Insufficient data" in stats["recommendation"] -def test_calculate_statistics__multi_variant_zero_total__returns_insufficient_data() -> None: +def test_calculate_statistics__multi_variant_zero_total__returns_insufficient_data() -> ( + None +): # Given results = { "blue": {"total": 100, "conversions": 10, "rate": 0.10},