fix #1867: use sample_size from config to build enrollments query (#1941

)
mozilla · Oct 3, 2023 · 33de27b · 33de27b
1 parent 004ea7a
commit 33de27b
Show file tree

Hide file tree

Showing 5 changed files with 35 additions and 15 deletions.
diff --git a/jetstream/analysis.py b/jetstream/analysis.py
@@ -462,6 +462,7 @@ def validate(self) -> None:
             None,
             exposure_signal,
             segments,
+            self.config.experiment.sample_size or None,
         )
 
         self._write_sql_output(
@@ -748,6 +749,7 @@ def enrollments_query(self, time_limits: TimeLimits) -> str:
             None,
             exposure_signal,
             segments,
+            self.config.experiment.sample_size or None,
         )
 
     def ensure_enrollments(self, current_date: datetime) -> None:

diff --git a/jetstream/tests/data/test_events.ndjson b/jetstream/tests/data/test_events.ndjson
@@ -1,7 +1,7 @@
-{"submission_date": "2020-04-02", "client_id": "aaaa", "event_category": "normandy", "event_method": "enroll", "event_string_value": "test-experiment", "event_map_values": [{"key": "branch", "value": "branch1"}]}
-{"submission_date": "2020-04-03", "client_id": "bbbb", "event_category": "normandy", "event_method": "enroll", "event_string_value": "test-experiment", "event_map_values": [{"key": "branch", "value": "branch2"}]}
-{"submission_date": "2020-04-02", "client_id": "aaaa", "event_category": "normandy", "event_method": "exposure", "event_string_value": "test-experiment", "event_map_values": [{"key": "branchSlug", "value": "branch1"}]}
-{"submission_date": "2020-04-03", "client_id": "bbbb", "event_category": "normandy", "event_method": "expose", "event_string_value": "test-experiment", "event_map_values": [{"key": "branchSlug", "value": "branch2"}]}
-{"submission_date": "2020-04-03", "client_id": "bbbb", "event_category": "foo"}
-{"submission_date": "2020-04-03", "client_id": "cccc", "event_category": "foo", "event_method": "enroll", "event_string_value": "test-experiment", "event_map_values": [{"key": "branch", "value": "branch1"}]}
-{"submission_date": "2020-04-03", "client_id": "dddd", "event_category": "foo", "event_method": "enroll", "event_string_value": "test-experiment", "event_map_values": [{"key": "branch", "value": "branch2"}]}
+{"submission_date": "2020-04-02", "client_id": "aaaa", "event_category": "normandy", "event_method": "enroll", "event_string_value": "test-experiment", "event_map_values": [{"key": "branch", "value": "branch1"}], "sample_id": 5}
+{"submission_date": "2020-04-03", "client_id": "bbbb", "event_category": "normandy", "event_method": "enroll", "event_string_value": "test-experiment", "event_map_values": [{"key": "branch", "value": "branch2"}], "sample_id": 20}
+{"submission_date": "2020-04-02", "client_id": "aaaa", "event_category": "normandy", "event_method": "exposure", "event_string_value": "test-experiment", "event_map_values": [{"key": "branchSlug", "value": "branch1"}], "sample_id": 35}
+{"submission_date": "2020-04-03", "client_id": "bbbb", "event_category": "normandy", "event_method": "expose", "event_string_value": "test-experiment", "event_map_values": [{"key": "branchSlug", "value": "branch2"}], "sample_id": 50}
+{"submission_date": "2020-04-03", "client_id": "bbbb", "event_category": "foo", "sample_id": 65}
+{"submission_date": "2020-04-03", "client_id": "cccc", "event_category": "foo", "event_method": "enroll", "event_string_value": "test-experiment", "event_map_values": [{"key": "branch", "value": "branch1"}], "sample_id": 80}
+{"submission_date": "2020-04-03", "client_id": "dddd", "event_category": "foo", "event_method": "enroll", "event_string_value": "test-experiment", "event_map_values": [{"key": "branch", "value": "branch2"}], "sample_id": 95}
diff --git a/jetstream/tests/test_cli.py b/jetstream/tests/test_cli.py
@@ -134,6 +134,24 @@ def test_validate_example_config(self, runner, monkeypatch):
             assert "Skipping example config" in result.output
             assert result.exit_code == 0
 
+    def test_validate_experiment_downsampling_config(self, runner, monkeypatch):
+        monkeypatch.setattr("jetstream.cli.ExperimentCollection.from_experimenter", cli_experiments)
+        with runner.isolated_filesystem():
+            conf = dedent(
+                """
+                [experiment]
+                enrollment_period = 7
+                sample_size = 10
+                """
+            )
+
+            with open("my_cool_experiment.toml", "w") as config:
+                config.write(conf)
+
+            result = runner.invoke(cli.validate_config, ["my_cool_experiment.toml"])
+
+            assert result.exit_code == 0
+
     def test_validate_private_example_config(self, runner, monkeypatch):
         monkeypatch.setattr("jetstream.cli.ExperimentCollection.from_experimenter", cli_experiments)
         with runner.isolated_filesystem():

diff --git a/requirements.in b/requirements.in
@@ -154,10 +154,10 @@ matplotlib==3.8.0
     # via mozanalysis
 mccabe==0.7.0
     # via flake8
-mozanalysis==2023.5.3
+mozanalysis==2023.9.4
     # via mozilla-jetstream
     # via -r -
-mozilla-metric-config-parser==2023.9.3
+mozilla-metric-config-parser==2023.9.4
     # via
     #   mozanalysis
     #   mozilla-jetstream

diff --git a/requirements.txt b/requirements.txt
@@ -812,13 +812,13 @@ mccabe==0.7.0 \
     # via
     #   -r requirements.in
     #   flake8
-mozanalysis==2023.5.3 \
-    --hash=sha256:3dfbe91e69a7460a7d8bf75c19ca2501d6e8160c4ad3a41d142f5601f1721b4b \
-    --hash=sha256:a9bdb4d3aab005f9765b8828aa0a96a5b04d80df279156f8fd84a6e711a7a879
+mozanalysis==2023.9.4 \
+    --hash=sha256:58aa4a499216148cb9b1861e1071d6087fa93d40206e9cae34f1c6793254010b \
+    --hash=sha256:a92092c73b7bc31a6fc4492c973b2a0b535b192c0b815cf4b3e31bda66e72649
     # via -r requirements.in
-mozilla-metric-config-parser==2023.9.3 \
-    --hash=sha256:17bedf09105ae509e4fdac60a22873a4b07b251b0f35bf79fdcdf3d2e2a6a9a3 \
-    --hash=sha256:313e62e32655895c94c2524d8262eb10e91b4c20a332c3935bfdf8c76d8779f3
+mozilla-metric-config-parser==2023.9.4 \
+    --hash=sha256:5751c59dd9937042b46ed99465183d2fa06b2bd2078aceabafed3eef97967516 \
+    --hash=sha256:ff4e420e1b77624bda07f0b283863ddcea26663c642fc6835902b07380616bd5
     # via
     #   -r requirements.in
     #   mozanalysis