diff --git a/README.md b/README.md index 57aac968..edef13e6 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,10 @@ Supported Python versions ---------- The SDK is tested with the most recent patch releases of Python 2.7, 3.3, 3.4, 3.5, and 3.6. Python 2.6 is no longer supported. +Using flag data from a file +--------------------------- +For testing purposes, the SDK can be made to read feature flag state from a file or files instead of connecting to LaunchDarkly. See [`file_data_source.py`](https://github.com/launchdarkly/python-client/blob/master/ldclient/file_data_source.py) for more details. + Learn more ----------- diff --git a/ldclient/file_data_source.py b/ldclient/file_data_source.py new file mode 100644 index 00000000..ebff765b --- /dev/null +++ b/ldclient/file_data_source.py @@ -0,0 +1,272 @@ +import json +import os +import six +import traceback + +have_yaml = False +try: + import yaml + have_yaml = True +except ImportError: + pass + +have_watchdog = False +try: + import watchdog + import watchdog.events + import watchdog.observers + have_watchdog = True +except ImportError: + pass + +from ldclient.interfaces import UpdateProcessor +from ldclient.repeating_timer import RepeatingTimer +from ldclient.util import log +from ldclient.versioned_data_kind import FEATURES, SEGMENTS + + +class FileDataSource(UpdateProcessor): + @classmethod + def factory(cls, **kwargs): + """Provides a way to use local files as a source of feature flag state. This would typically be + used in a test environment, to operate using a predetermined feature flag state without an + actual LaunchDarkly connection. + + To use this component, call `FileDataSource.factory`, and store its return value in the + `update_processor_class` property of your LaunchDarkly client configuration. In the options + to `factory`, set `paths` to the file path(s) of your data file(s): + :: + + factory = FileDataSource.factory(paths = [ myFilePath ]) + config = Config(update_processor_class = factory) + + This will cause the client not to connect to LaunchDarkly to get feature flags. The + client may still make network connections to send analytics events, unless you have disabled + this with Config.send_events or Config.offline. + + Flag data files can be either JSON or YAML (in order to use YAML, you must install the 'pyyaml' + package). They contain an object with three possible properties: + + * "flags": Feature flag definitions. + * "flagValues": Simplified feature flags that contain only a value. + * "segments": User segment definitions. + + The format of the data in "flags" and "segments" is defined by the LaunchDarkly application + and is subject to change. Rather than trying to construct these objects yourself, it is simpler + to request existing flags directly from the LaunchDarkly server in JSON format, and use this + output as the starting point for your file. In Linux you would do this: + :: + + curl -H "Authorization: {your sdk key}" https://app.launchdarkly.com/sdk/latest-all + + The output will look something like this (but with many more properties): + :: + + { + "flags": { + "flag-key-1": { + "key": "flag-key-1", + "on": true, + "variations": [ "a", "b" ] + } + }, + "segments": { + "segment-key-1": { + "key": "segment-key-1", + "includes": [ "user-key-1" ] + } + } + } + + Data in this format allows the SDK to exactly duplicate all the kinds of flag behavior supported + by LaunchDarkly. However, in many cases you will not need this complexity, but will just want to + set specific flag keys to specific values. For that, you can use a much simpler format: + :: + + { + "flagValues": { + "my-string-flag-key": "value-1", + "my-boolean-flag-key": true, + "my-integer-flag-key": 3 + } + } + + Or, in YAML: + :: + + flagValues: + my-string-flag-key: "value-1" + my-boolean-flag-key: true + my-integer-flag-key: 1 + + It is also possible to specify both "flags" and "flagValues", if you want some flags + to have simple values and others to have complex behavior. However, it is an error to use the + same flag key or segment key more than once, either in a single file or across multiple files. + + If the data source encounters any error in any file-- malformed content, a missing file, or a + duplicate key-- it will not load flags from any of the files. + + :param kwargs: + See below + + :Keyword arguments: + * **paths** (array): The paths of the source files for loading flag data. These may be absolute paths + or relative to the current working directory. Files will be parsed as JSON unless the 'pyyaml' + package is installed, in which case YAML is also allowed. + * **auto_update** (boolean): True if the data source should watch for changes to the source file(s) + and reload flags whenever there is a change. The default implementation of this feature is based on + polling the filesystem, which may not perform well; if you install the 'watchdog' package (not + included by default, to avoid adding unwanted dependencies to the SDK), its native file watching + mechanism will be used instead. Note that auto-updating will only work if all of the files you + specified have valid directory paths at startup time. + * **poll_interval** (float): The minimum interval, in seconds, between checks for file modifications - + used only if auto_update is true, and if the native file-watching mechanism from 'watchdog' is not + being used. The default value is 1 second. + """ + return lambda config, store, ready : FileDataSource(store, kwargs, ready) + + def __init__(self, store, options, ready): + self._store = store + self._ready = ready + self._inited = False + self._paths = options.get('paths', []) + if isinstance(self._paths, six.string_types): + self._paths = [ self._paths ] + self._auto_update = options.get('auto_update', False) + self._auto_updater = None + self._poll_interval = options.get('poll_interval', 1) + self._force_polling = options.get('force_polling', False) # used only in tests + + def start(self): + self._load_all() + + if self._auto_update: + self._auto_updater = self._start_auto_updater() + + # We will signal readiness immediately regardless of whether the file load succeeded or failed - + # the difference can be detected by checking initialized() + self._ready.set() + + def stop(self): + if self._auto_updater: + self._auto_updater.stop() + + def initialized(self): + return self._inited + + def _load_all(self): + all_data = { FEATURES: {}, SEGMENTS: {} } + for path in self._paths: + try: + self._load_file(path, all_data) + except Exception as e: + log.error('Unable to load flag data from "%s": %s' % (path, repr(e))) + traceback.print_exc() + return + self._store.init(all_data) + self._inited = True + + def _load_file(self, path, all_data): + content = None + with open(path, 'r') as f: + content = f.read() + parsed = self._parse_content(content) + for key, flag in six.iteritems(parsed.get('flags', {})): + self._add_item(all_data, FEATURES, flag) + for key, value in six.iteritems(parsed.get('flagValues', {})): + self._add_item(all_data, FEATURES, self._make_flag_with_value(key, value)) + for key, segment in six.iteritems(parsed.get('segments', {})): + self._add_item(all_data, SEGMENTS, segment) + + def _parse_content(self, content): + if have_yaml: + return yaml.load(content) # pyyaml correctly parses JSON too + return json.loads(content) + + def _add_item(self, all_data, kind, item): + items = all_data[kind] + key = item.get('key') + if items.get(key) is None: + items[key] = item + else: + raise Exception('In %s, key "%s" was used more than once' % (kind.namespace, key)) + + def _make_flag_with_value(self, key, value): + return { + 'key': key, + 'on': True, + 'fallthrough': { + 'variation': 0 + }, + 'variations': [ value ] + } + + def _start_auto_updater(self): + resolved_paths = [] + for path in self._paths: + try: + resolved_paths.append(os.path.realpath(path)) + except: + log.warn('Cannot watch for changes to data file "%s" because it is an invalid path' % path) + if have_watchdog and not self._force_polling: + return FileDataSource.WatchdogAutoUpdater(resolved_paths, self._load_all) + else: + return FileDataSource.PollingAutoUpdater(resolved_paths, self._load_all, self._poll_interval) + + # Watch for changes to data files using the watchdog package. This uses native OS filesystem notifications + # if available for the current platform. + class WatchdogAutoUpdater(object): + def __init__(self, resolved_paths, reloader): + watched_files = set(resolved_paths) + + class LDWatchdogHandler(watchdog.events.FileSystemEventHandler): + def on_any_event(self, event): + if event.src_path in watched_files: + reloader() + + dir_paths = set() + for path in resolved_paths: + dir_paths.add(os.path.dirname(path)) + + self._observer = watchdog.observers.Observer() + handler = LDWatchdogHandler() + for path in dir_paths: + self._observer.schedule(handler, path) + self._observer.start() + + def stop(self): + self._observer.stop() + self._observer.join() + + # Watch for changes to data files by polling their modification times. This is used if auto-update is + # on but the watchdog package is not installed. + class PollingAutoUpdater(object): + def __init__(self, resolved_paths, reloader, interval): + self._paths = resolved_paths + self._reloader = reloader + self._file_times = self._check_file_times() + self._timer = RepeatingTimer(interval, self._poll) + self._timer.start() + + def stop(self): + self._timer.stop() + + def _poll(self): + new_times = self._check_file_times() + changed = False + for file_path, file_time in six.iteritems(self._file_times): + if new_times.get(file_path) is not None and new_times.get(file_path) != file_time: + changed = True + break + self._file_times = new_times + if changed: + self._reloader() + + def _check_file_times(self): + ret = {} + for path in self._paths: + try: + ret[path] = os.path.getmtime(path) + except: + ret[path] = None + return ret diff --git a/test-requirements.txt b/test-requirements.txt index ee547312..413ef355 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,4 +4,6 @@ redis>=2.10.5 coverage>=4.4 pytest-capturelog>=0.7 pytest-cov>=2.4.0 -codeclimate-test-reporter>=0.2.1 \ No newline at end of file +codeclimate-test-reporter>=0.2.1 +pyyaml>=3.0 +watchdog>=0.9 diff --git a/testing/test_file_data_source.py b/testing/test_file_data_source.py new file mode 100644 index 00000000..68d1e5b7 --- /dev/null +++ b/testing/test_file_data_source.py @@ -0,0 +1,239 @@ +import json +import os +import pytest +import six +import tempfile +import threading +import time + +from ldclient.client import LDClient +from ldclient.config import Config +from ldclient.feature_store import InMemoryFeatureStore +from ldclient.file_data_source import FileDataSource +from ldclient.versioned_data_kind import FEATURES, SEGMENTS + + +all_flag_keys = [ 'flag1', 'flag2' ] +all_properties_json = ''' + { + "flags": { + "flag1": { + "key": "flag1", + "on": true, + "fallthrough": { + "variation": 2 + }, + "variations": [ "fall", "off", "on" ] + } + }, + "flagValues": { + "flag2": "value2" + }, + "segments": { + "seg1": { + "key": "seg1", + "include": ["user1"] + } + } + } +''' + +all_properties_yaml = ''' +--- +flags: + flag1: + key: flag1 + "on": true +flagValues: + flag2: value2 +segments: + seg1: + key: seg1 + include: ["user1"] +''' + +flag_only_json = ''' + { + "flags": { + "flag1": { + "key": "flag1", + "on": true, + "fallthrough": { + "variation": 2 + }, + "variations": [ "fall", "off", "on" ] + } + } + } +''' + +segment_only_json = ''' + { + "segments": { + "seg1": { + "key": "seg1", + "include": ["user1"] + } + } + } +''' + +data_source = None +store = None +ready = None + + +def setup_function(): + global data_source, store, ready + store = InMemoryFeatureStore() + ready = threading.Event() + +def teardown_function(): + if data_source is not None: + data_source.stop() + +def make_data_source(**kwargs): + global data_source + data_source = FileDataSource.factory(**kwargs)(Config(), store, ready) + return data_source + +def make_temp_file(content): + f, path = tempfile.mkstemp() + os.write(f, six.b(content)) + os.close(f) + return path + +def replace_file(path, content): + with open(path, 'w') as f: + f.write(content) + +def test_does_not_load_data_prior_to_start(): + path = make_temp_file('{"flagValues":{"key":"value"}}') + try: + source = make_data_source(paths = path) + assert ready.is_set() is False + assert source.initialized() is False + assert store.initialized is False + finally: + os.remove(path) + +def test_loads_flags_on_start_from_json(): + path = make_temp_file(all_properties_json) + try: + source = make_data_source(paths = path) + source.start() + assert store.initialized is True + assert sorted(list(store.all(FEATURES, lambda x: x).keys())) == all_flag_keys + finally: + os.remove(path) + +def test_loads_flags_on_start_from_yaml(): + path = make_temp_file(all_properties_yaml) + try: + source = make_data_source(paths = path) + source.start() + assert store.initialized is True + assert sorted(list(store.all(FEATURES, lambda x: x).keys())) == all_flag_keys + finally: + os.remove(path) + +def test_sets_ready_event_and_initialized_on_successful_load(): + path = make_temp_file(all_properties_json) + try: + source = make_data_source(paths = path) + source.start() + assert source.initialized() is True + assert ready.is_set() is True + finally: + os.remove(path) + +def test_sets_ready_event_and_does_not_set_initialized_on_unsuccessful_load(): + bad_file_path = 'no-such-file' + source = make_data_source(paths = bad_file_path) + source.start() + assert source.initialized() is False + assert ready.is_set() is True + +def test_can_load_multiple_files(): + path1 = make_temp_file(flag_only_json) + path2 = make_temp_file(segment_only_json) + try: + source = make_data_source(paths = [ path1, path2 ]) + source.start() + assert len(store.all(FEATURES, lambda x: x)) == 1 + assert len(store.all(SEGMENTS, lambda x: x)) == 1 + finally: + os.remove(path1) + os.remove(path2) + +def test_does_not_allow_duplicate_keys(): + path1 = make_temp_file(flag_only_json) + path2 = make_temp_file(flag_only_json) + try: + source = make_data_source(paths = [ path1, path2 ]) + source.start() + assert len(store.all(FEATURES, lambda x: x)) == 0 + finally: + os.remove(path1) + os.remove(path2) + +def test_does_not_reload_modified_file_if_auto_update_is_off(): + path = make_temp_file(flag_only_json) + try: + source = make_data_source(paths = path) + source.start() + assert len(store.all(SEGMENTS, lambda x: x)) == 0 + time.sleep(0.5) + replace_file(path, segment_only_json) + time.sleep(0.5) + assert len(store.all(SEGMENTS, lambda x: x)) == 0 + finally: + os.remove(path) + +def do_auto_update_test(options): + path = make_temp_file(flag_only_json) + options['paths'] = path + try: + source = make_data_source(**options) + source.start() + assert len(store.all(SEGMENTS, lambda x: x)) == 0 + time.sleep(0.5) + replace_file(path, segment_only_json) + deadline = time.time() + 10 + while time.time() < deadline: + time.sleep(0.1) + if len(store.all(SEGMENTS, lambda x: x)) == 1: + return + assert False, "Flags were not reloaded after 10 seconds" + finally: + os.remove(path) + +def test_reloads_modified_file_if_auto_update_is_on(): + do_auto_update_test({ 'auto_update': True }) + +def test_reloads_modified_file_in_polling_mode(): + do_auto_update_test({ 'auto_update': True, 'force_polling': True, 'poll_interval': 0.1 }) + +def test_evaluates_full_flag_with_client_as_expected(): + path = make_temp_file(all_properties_json) + try: + factory = FileDataSource.factory(paths = path) + client = LDClient(config=Config(update_processor_class = factory, send_events = False)) + value = client.variation('flag1', { 'key': 'user' }, '') + assert value == 'on' + finally: + os.remove(path) + if client is not None: + client.close() + +def test_evaluates_simplified_flag_with_client_as_expected(): + path = make_temp_file(all_properties_json) + try: + factory = FileDataSource.factory(paths = path) + client = LDClient(config=Config(update_processor_class = factory, send_events = False)) + value = client.variation('flag2', { 'key': 'user' }, '') + assert value == 'value2' + finally: + os.remove(path) + if client is not None: + client.close()