From d8970f7c9a545a207049d22674298219b1efc7e4 Mon Sep 17 00:00:00 2001 From: Qingzhuo Zhen Date: Fri, 12 Aug 2022 17:02:37 -0700 Subject: [PATCH 1/2] feat: testing, comments and prepare for release --- MANIFEST.in | 1 + README.md | 37 +++++-- setup.py | 1 + src/amplitude_experiment/local/__init__.py | 0 src/amplitude_experiment/local/client.py | 41 +++++-- src/amplitude_experiment/local/config.py | 24 ++++- .../local/evaluation/__init__.py | 1 + src/amplitude_experiment/local/poller.py | 3 + src/amplitude_experiment/remote/__init__.py | 0 src/amplitude_experiment/remote/config.py | 2 +- tests/local/benchmark_test.py | 102 ++++++++++++++++++ tests/local/client_test.py | 9 +- 12 files changed, 202 insertions(+), 19 deletions(-) create mode 100644 MANIFEST.in create mode 100644 src/amplitude_experiment/local/__init__.py create mode 100644 src/amplitude_experiment/local/evaluation/__init__.py create mode 100644 src/amplitude_experiment/remote/__init__.py create mode 100644 tests/local/benchmark_test.py diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..b4c0af7 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +graft src/amplitude_experiment diff --git a/README.md b/README.md index 613d2f9..cc9a36e 100644 --- a/README.md +++ b/README.md @@ -15,20 +15,24 @@ Amplitude Python Server SDK for Experiment. pip install amplitude-experiment ``` -## Quick Start +## Remote Evaluation Quick Start ```python -from amplitude_experiment import Experiment, Config, Client, User +from amplitude_experiment import Experiment, RemoteEvaluationConfig, RemoteEvaluationClient, User # (1) Get your deployment's API key apiKey = 'YOUR-API-KEY' - # (2) Initialize the experiment client -experiment = Experiment.initialize(api_key) + # (2) Initialize the experiment remote evaluation +experiment = Experiment.initialize_remote(api_key) # (3) Fetch variants for a user -user = User(device_id="abcdefg", user_id="user@company.com", user_properties={ - 'premium': True -}) +user = User( + device_id="abcdefg", + user_id="user@company.com", + user_properties={ + 'premium': True + } +) # (4) Lookup a flag's variant # @@ -52,8 +56,27 @@ def fetch_callback(user, variants): else: # Flag is off +``` + +## Local Evaluation Quick Start +```python +# (1) Initialize the local evaluation client with a server deployment key. +experiment = Experiment.initialize_local(api_key) +# (2) Start the local evaluation client. +experiment.start() + +# (3) Evaluate a user. +user = User( + device_id="abcdefg", + user_id="user@company.com", + user_properties={ + 'premium': True + } +) +variants = experiment.evaluate(user) ``` + ## More Information Please visit our :100:[Developer Center](https://www.docs.developers.amplitude.com/experiment/sdks/python-sdk/) for more instructions on using our the SDK. diff --git a/setup.py b/setup.py index d311aba..08319b4 100644 --- a/setup.py +++ b/setup.py @@ -35,6 +35,7 @@ keywords="amplitude, python, backend", package_dir={"": "src"}, packages=["amplitude_experiment"], + include_package_data=True, python_requires=">=3.6, <4", license='MIT License', project_urls={ diff --git a/src/amplitude_experiment/local/__init__.py b/src/amplitude_experiment/local/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/amplitude_experiment/local/client.py b/src/amplitude_experiment/local/client.py index 115a375..16b5a53 100644 --- a/src/amplitude_experiment/local/client.py +++ b/src/amplitude_experiment/local/client.py @@ -1,7 +1,7 @@ import json import logging from threading import Lock -from typing import Any, List +from typing import Any, List, Dict from .config import LocalEvaluationConfig from ..user import User @@ -12,7 +12,18 @@ class LocalEvaluationClient: - def __init__(self, api_key, config=None): + """Experiment client for evaluating variants for a user locally.""" + + def __init__(self, api_key: str, config : LocalEvaluationConfig = None): + """ + Creates a new Experiment LocalEvaluationClient instance. + Parameters: + api_key (str): The environment API Key + config (LocalEvaluationConfig): Config Object + + Returns: + Experiment Client instance. + """ if not api_key: raise ValueError("Experiment API key is empty") self.api_key = api_key @@ -25,12 +36,29 @@ def __init__(self, api_key, config=None): self.rules = {} self.poller = Poller(self.config.flag_config_polling_interval_millis / 1000, self.__do_rules) self.lock = Lock() + self.is_running = False def start(self): + """ + Fetch initial flag configurations and start polling for updates. You must call this function to begin + polling for flag config updates. + """ + if self.is_running: + return + self.is_running = True self.__do_rules() self.poller.start() - def evaluate(self, user: User, flag_keys: List[str] = None): + def evaluate(self, user: User, flag_keys: List[str] = None) -> Dict[str, Variant]: + """ + Locally evaluates flag variants for a user. + Parameters: + user (User): The user to evaluate + flag_keys (List[str]): The flags to evaluate with the user. If empty, all flags from the flag cache are evaluated. + + Returns: + The evaluated variants. + """ no_flag_keys = flag_keys is None or len(flag_keys) == 0 rules = [] for key, value in self.rules.items(): @@ -59,10 +87,10 @@ def __do_rules(self): body = None self.logger.debug('[Experiment] Get flag configs') try: - response = conn.request('POST', '/sdk/rules?eval_mode=local', body, headers) + response = conn.request('GET', '/sdk/rules?eval_mode=local', body, headers) response_body = response.read().decode("utf8") if response.status != 200: - raise Exception(f"flagConfigs - received error response: ${response.status}: ${response_body}") + raise Exception(f"[Experiment] Get flagConfigs - received error response: ${response.status}: ${response_body}") self.logger.debug(f"[Experiment] Got flag configs: {response_body}") parsed_rules = self.__parse(json.loads(response_body)) self.lock.acquire() @@ -85,10 +113,11 @@ def __setup_connection_pool(self): def close(self) -> None: """ - Close resource like connection pool with client + Stop polling for flag configurations. Close resource like connection pool with client """ self.poller.stop() self._connection_pool.close() + self.is_running = False def __enter__(self) -> 'LocalEvaluationClient': return self diff --git a/src/amplitude_experiment/local/config.py b/src/amplitude_experiment/local/config.py index bd8039d..b797a7b 100644 --- a/src/amplitude_experiment/local/config.py +++ b/src/amplitude_experiment/local/config.py @@ -1,10 +1,26 @@ class LocalEvaluationConfig: + """Experiment Local Client Configuration""" + DEFAULT_SERVER_URL = 'https://api.lab.amplitude.com' - def __init__(self, debug=False, - server_url=DEFAULT_SERVER_URL, - flag_config_polling_interval_millis=30000, - flag_config_poller_request_timeout_millis=10000): + def __init__(self, debug: bool = False, + server_url: str = DEFAULT_SERVER_URL, + flag_config_polling_interval_millis: int = 30000, + flag_config_poller_request_timeout_millis: int = 10000): + """ + Initialize a config + Parameters: + debug (bool): Set to true to log some extra information to the console. + server_url (str): The server endpoint from which to request variants. + flag_config_polling_interval_millis (int): The interval in milliseconds to poll the amplitude server for + flag config updates. These rules are stored in memory and used when calling evaluate() + to perform local evaluation. + flag_config_poller_request_timeout_millis (int): The request timeout, in milliseconds, + used when fetching variants. + + Returns: + The config object + """ self.debug = debug self.server_url = server_url self.flag_config_polling_interval_millis = flag_config_polling_interval_millis diff --git a/src/amplitude_experiment/local/evaluation/__init__.py b/src/amplitude_experiment/local/evaluation/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/amplitude_experiment/local/evaluation/__init__.py @@ -0,0 +1 @@ + diff --git a/src/amplitude_experiment/local/poller.py b/src/amplitude_experiment/local/poller.py index 8788cab..a939440 100644 --- a/src/amplitude_experiment/local/poller.py +++ b/src/amplitude_experiment/local/poller.py @@ -3,6 +3,9 @@ class Poller: + """ + Poller to run a function every interval + """ def __init__(self, interval, function, *args, **kwargs): self._timer = None self.interval = interval diff --git a/src/amplitude_experiment/remote/__init__.py b/src/amplitude_experiment/remote/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/amplitude_experiment/remote/config.py b/src/amplitude_experiment/remote/config.py index 6ce1707..7e84bf5 100644 --- a/src/amplitude_experiment/remote/config.py +++ b/src/amplitude_experiment/remote/config.py @@ -1,5 +1,5 @@ class RemoteEvaluationConfig: - """Experiment Client Configuration""" + """Experiment Remote Client Configuration""" DEFAULT_SERVER_URL = 'https://api.lab.amplitude.com' diff --git a/tests/local/benchmark_test.py b/tests/local/benchmark_test.py new file mode 100644 index 0000000..e27815a --- /dev/null +++ b/tests/local/benchmark_test.py @@ -0,0 +1,102 @@ +import random +import time +import unittest + +from src.amplitude_experiment import LocalEvaluationClient, User + +API_KEY = 'server-qz35UwzJ5akieoAdIgzM4m9MIiOLXLoz' + + +def random_boolean(): + return bool(random.getrandbits(1)) + + +def measure(function, *args, **kwargs): + start = time.time() + function(*args, **kwargs) + elapsed = (time.time() - start) * 1000 + return elapsed + + +def random_string(length): + letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789' + return ''.join(random.choice(letters) for i in range(length)) + + +def random_experiment_user(): + n = 15 + user = User(user_id=random_string(n)) + if random_boolean(): + user.device_id = random_string(n) + if random_boolean(): + user.platform = random_string(n) + if random_boolean(): + user.version = random_string(n) + if random_boolean(): + user.os = random_string(n) + if random_boolean(): + user.device_manufacturer = random_string(n) + if random_boolean(): + user.device_model = random_string(n) + if random_boolean(): + user.device_brand = random_string(n) + if random_boolean(): + user.user_properties = { + 'test': 'test' + } + return user + + +def random_benchmark_flag(): + n = random.randint(1, 4) + return f"local-evaluation-benchmark-{n}" + + +class BenchmarkTestCase(unittest.TestCase): + _local_evaluation_client: LocalEvaluationClient = None + + @classmethod + def setUpClass(cls) -> None: + cls._local_evaluation_client = LocalEvaluationClient(API_KEY) + cls._local_evaluation_client.start() + + @classmethod + def tearDownClass(cls) -> None: + cls._local_evaluation_client.close() + + def test_evaluate_benchmark_1_flag_smaller_than_10_ms(self): + user = random_experiment_user() + flag = random_benchmark_flag() + duration = measure(self._local_evaluation_client.evaluate, user, [flag]) + self.assertTrue(duration < 10) + + def test_evaluate_benchmark_10_flag_smaller_than_10_ms(self): + total = 0 + for i in range(10): + user = random_experiment_user() + flag = random_benchmark_flag() + duration = measure(self._local_evaluation_client.evaluate, user, [flag]) + total += duration + self.assertTrue(total < 10) + + def test_evaluate_benchmark_100_flag_smaller_than_100_ms(self): + total = 0 + for i in range(100): + user = random_experiment_user() + flag = random_benchmark_flag() + duration = measure(self._local_evaluation_client.evaluate, user, [flag]) + total += duration + self.assertTrue(total < 100) + + def test_evaluate_benchmark_1000_flag_smaller_than_1000_ms(self): + total = 0 + for i in range(1000): + user = random_experiment_user() + flag = random_benchmark_flag() + duration = measure(self._local_evaluation_client.evaluate, user, [flag]) + total += duration + self.assertTrue(total < 1000) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/local/client_test.py b/tests/local/client_test.py index 9af896d..28ae7e3 100644 --- a/tests/local/client_test.py +++ b/tests/local/client_test.py @@ -6,7 +6,6 @@ class LocalEvaluationClientTestCase(unittest.TestCase): - _local_evaluation_client: LocalEvaluationClient = None @classmethod @@ -18,6 +17,9 @@ def setUpClass(cls) -> None: def tearDownClass(cls) -> None: cls._local_evaluation_client.close() + def test_initialize_raise_error(self): + self.assertRaises(ValueError, LocalEvaluationClient, "") + def test_evaluate_all_flags_success(self): variants = self._local_evaluation_client.evaluate(test_user) expected_variant = Variant('on', 'payload') @@ -28,6 +30,11 @@ def test_evaluate_one_flag_success(self): expected_variant = Variant('on', 'payload') self.assertEqual(expected_variant, variants.get('sdk-local-evaluation-ci-test')) + def test_invalid_api_key_throw_exception(self): + invalid_local_api_key = 'client-DvWljIjiiuqLbyjqdvBaLFfEBrAvGuA3' + with LocalEvaluationClient(invalid_local_api_key) as test_client: + self.assertRaises(Exception, test_client.start, "[Experiment] Get flagConfigs - received error response") + if __name__ == '__main__': unittest.main() From 7a03eda0bdb17e0abd145ea37cda6950e055c000 Mon Sep 17 00:00:00 2001 From: Qingzhuo Zhen Date: Mon, 15 Aug 2022 12:11:10 -0700 Subject: [PATCH 2/2] resolve comments --- src/amplitude_experiment/connection_pool.py | 2 +- src/amplitude_experiment/local/client.py | 9 ++------- tests/factory_test.py | 2 +- tests/local/benchmark_test.py | 2 +- tests/local/client_test.py | 2 +- 5 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/amplitude_experiment/connection_pool.py b/src/amplitude_experiment/connection_pool.py index 753fd8b..6ad75db 100644 --- a/src/amplitude_experiment/connection_pool.py +++ b/src/amplitude_experiment/connection_pool.py @@ -130,7 +130,7 @@ def close(self) -> None: self.stop_clear_conn() pool, self._pool = self._pool, None for conn in pool: - conn.close() + conn.stop() def clear_idle_conn(self) -> None: if self.is_closed: diff --git a/src/amplitude_experiment/local/client.py b/src/amplitude_experiment/local/client.py index 16b5a53..aa7dad6 100644 --- a/src/amplitude_experiment/local/client.py +++ b/src/amplitude_experiment/local/client.py @@ -36,16 +36,12 @@ def __init__(self, api_key: str, config : LocalEvaluationConfig = None): self.rules = {} self.poller = Poller(self.config.flag_config_polling_interval_millis / 1000, self.__do_rules) self.lock = Lock() - self.is_running = False def start(self): """ Fetch initial flag configurations and start polling for updates. You must call this function to begin polling for flag config updates. """ - if self.is_running: - return - self.is_running = True self.__do_rules() self.poller.start() @@ -111,16 +107,15 @@ def __setup_connection_pool(self): self._connection_pool = HTTPConnectionPool(host, max_size=1, idle_timeout=30, read_timeout=timeout, scheme=scheme) - def close(self) -> None: + def stop(self) -> None: """ Stop polling for flag configurations. Close resource like connection pool with client """ self.poller.stop() self._connection_pool.close() - self.is_running = False def __enter__(self) -> 'LocalEvaluationClient': return self def __exit__(self, *exit_info: Any) -> None: - self.close() + self.stop() diff --git a/tests/factory_test.py b/tests/factory_test.py index a17dee6..b7710e1 100644 --- a/tests/factory_test.py +++ b/tests/factory_test.py @@ -15,7 +15,7 @@ def test_singleton_local_instance(self): client1 = Experiment.initialize_local(API_KEY) client2 = Experiment.initialize_local(API_KEY) self.assertEqual(client1, client2) - client1.close() + client1.stop() if __name__ == '__main__': diff --git a/tests/local/benchmark_test.py b/tests/local/benchmark_test.py index e27815a..f450e20 100644 --- a/tests/local/benchmark_test.py +++ b/tests/local/benchmark_test.py @@ -62,7 +62,7 @@ def setUpClass(cls) -> None: @classmethod def tearDownClass(cls) -> None: - cls._local_evaluation_client.close() + cls._local_evaluation_client.stop() def test_evaluate_benchmark_1_flag_smaller_than_10_ms(self): user = random_experiment_user() diff --git a/tests/local/client_test.py b/tests/local/client_test.py index 28ae7e3..d963a28 100644 --- a/tests/local/client_test.py +++ b/tests/local/client_test.py @@ -15,7 +15,7 @@ def setUpClass(cls) -> None: @classmethod def tearDownClass(cls) -> None: - cls._local_evaluation_client.close() + cls._local_evaluation_client.stop() def test_initialize_raise_error(self): self.assertRaises(ValueError, LocalEvaluationClient, "")