From efe3c101db03eb73a5b4e4252b4cfeb5f072a323 Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Fri, 6 Feb 2015 12:47:43 +0200 Subject: [PATCH 01/17] Add delivery classes to contacts api --- go_contacts/backends/contacts.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/go_contacts/backends/contacts.py b/go_contacts/backends/contacts.py index 9bb0fc1..9ca18c7 100644 --- a/go_contacts/backends/contacts.py +++ b/go_contacts/backends/contacts.py @@ -1,6 +1,7 @@ """ Riak contacts backend and collection. """ +from collections import defaultdict from twisted.internet.defer import inlineCallbacks, returnValue from zope.interface import implementer @@ -9,6 +10,7 @@ from go.vumitools.contact import ( ContactStore, ContactNotFoundError, Contact) +from go.vumitools.contact.models import DELIVERY_CLASSES from go_api.collections import ICollection from go_api.collections.errors import ( @@ -52,6 +54,7 @@ class RiakContactsCollection(object): def __init__(self, contact_store, max_contacts_per_page): self.contact_store = contact_store self.max_contacts_per_page = max_contacts_per_page + self.delivery_classes = self._reverse_delivery_class() @staticmethod def _pick_fields(data, keys): @@ -91,6 +94,12 @@ def all_keys(self): """ raise NotImplementedError() + def _reverse_delivery_class(self): + res = defaultdict(list) + for cls, value in DELIVERY_CLASSES.iteritems(): + res[value['field']].append(cls) + return res + def stream(self, query): """ Return a :class:`PausingDeferredQueue` of the objects in the From ba166187938e37862ae38ab07b38bd2b58146a43 Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Fri, 6 Feb 2015 12:48:30 +0200 Subject: [PATCH 02/17] Added ability to get contacts by parameter query --- go_contacts/backends/contacts.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/go_contacts/backends/contacts.py b/go_contacts/backends/contacts.py index 9ca18c7..0767656 100644 --- a/go_contacts/backends/contacts.py +++ b/go_contacts/backends/contacts.py @@ -100,6 +100,32 @@ def _reverse_delivery_class(self): res[value['field']].append(cls) return res + @inlineCallbacks + def _get_contacts_by_query(self, query): + try: + [field, value] = query.split('=') + except ValueError: + raise CollectionUsageError( + "Query must be of the form 'field=value'") + valid_keys = self.delivery_classes.keys() + if field not in valid_keys: + raise CollectionUsageError( + "Query field must be one of: %s" % valid_keys) + + contacts = [] + for cls in self.delivery_classes[field]: + try: + contact = ( + yield self.contact_store.contacts.contact_for_addr( + cls, value, create=False)) + contacts.append(contact) + except ContactNotFoundError: + pass + if not contacts: + raise ContactNotFoundError( + 'Contact with %s %s not found' % (field, value)) + returnValue(contacts) + def stream(self, query): """ Return a :class:`PausingDeferredQueue` of the objects in the @@ -109,8 +135,7 @@ def stream(self, query): :param unicode query: Search term requested through the API. Defaults to ``None`` if no - search term was requested. Currently not implemented and will raise - a CollectionUsageError if not ``None``. + search term was requested. Query must be of the form `field=value`. """ if query is not None: raise CollectionUsageError("query parameter not supported") @@ -155,7 +180,8 @@ def page(self, cursor, max_results, query): :rtype: tuple """ if query is not None: - raise CollectionUsageError("query parameter not supported") + contacts = yield self._get_contacts_by_query(query) + returnValue((None, contacts)) max_results = max_results or float('inf') max_results = min(max_results, self.max_contacts_per_page) From dcdd7cff59dfebfffa102bd844ced9978a2958de Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Fri, 6 Feb 2015 12:48:57 +0200 Subject: [PATCH 03/17] Added tests for invalid query parameters and formats --- .../tests/server_contacts_test_mixin.py | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/go_contacts/tests/server_contacts_test_mixin.py b/go_contacts/tests/server_contacts_test_mixin.py index e9131ee..568e77f 100644 --- a/go_contacts/tests/server_contacts_test_mixin.py +++ b/go_contacts/tests/server_contacts_test_mixin.py @@ -407,13 +407,30 @@ def test_page_bad_cursor(self): u"Riak error, possible invalid cursor: u'bad-id'") @inlineCallbacks - def test_page_query(self): + def test_page_invalid_query_format(self): """ - If a query parameter is supplied, a CollectionUsageError should be - thrown, as querys are not yet supported. + If an invalid query format is supplied, a CollectionUsageError should + be thrown. """ api = self.mk_api() code, data = yield self.request(api, 'GET', '/contacts/?query=foo') self.assertEqual(code, 400) self.assertEqual(data.get(u'status_code'), 400) - self.assertEqual(data.get(u'reason'), u'query parameter not supported') + self.assertEqual( + data.get(u'reason'), u"Query must be of the form 'field=value'") + + @inlineCallbacks + def test_page_invalid_query_parameter(self): + """ + If an invalid query parameter is supplied, a CollectionUsageError + should be thrown. + """ + api = self.mk_api() + code, data = yield self.request( + api, 'GET', '/contacts/?query="foo=bar"') + self.assertEqual(code, 400) + self.assertEqual(data.get(u'status_code'), 400) + self.assertEqual( + data.get(u'reason'), + u"Query field must be one of: ['msisdn', 'wechat_id', 'gtalk_id'," + " 'twitter_handle', 'mxit_id']") From 6d24cd1a348929b983f8933059fd7235606785ed Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Fri, 6 Feb 2015 12:53:25 +0200 Subject: [PATCH 04/17] Added test for getting page with query --- go_contacts/tests/server_contacts_test_mixin.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/go_contacts/tests/server_contacts_test_mixin.py b/go_contacts/tests/server_contacts_test_mixin.py index 568e77f..054ae37 100644 --- a/go_contacts/tests/server_contacts_test_mixin.py +++ b/go_contacts/tests/server_contacts_test_mixin.py @@ -434,3 +434,17 @@ def test_page_invalid_query_parameter(self): data.get(u'reason'), u"Query field must be one of: ['msisdn', 'wechat_id', 'gtalk_id'," " 'twitter_handle', 'mxit_id']") + + @inlineCallbacks + def test_page_with_query(self): + """ + If a valid query is supplied, the contact should be returned + """ + api = self.mk_api() + contact = yield self.create_contact(api, name=u'Bob', msisdn=u'+12345') + + code, data = yield self.request( + api, 'GET', '/contacts/?q="msisdn=+12345"') + self.assertEqual(code, 200) + self.assertEqual(data.get(u'cursor'), None) + self.assertEqual(data.get('data'), [contact]) From 7592fef9d4fb673ec21cc00ff34aefb6a895c830 Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Fri, 6 Feb 2015 13:11:03 +0200 Subject: [PATCH 05/17] Added support to FakeGoContacts for parameter queries --- verified-fake/fake_go_contacts.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/verified-fake/fake_go_contacts.py b/verified-fake/fake_go_contacts.py index 2fa151a..3203e1b 100644 --- a/verified-fake/fake_go_contacts.py +++ b/verified-fake/fake_go_contacts.py @@ -95,6 +95,8 @@ class FakeContacts(object): def __init__(self, contacts_data={}, max_contacts_per_page=10): self.contacts_data = contacts_data self.max_contacts_per_page = max_contacts_per_page + self.valid_search_keys = [ + 'msisdn', 'wechat_id', 'gtalk_id', 'twitter_handle', 'mxit_id'] @staticmethod def make_contact_dict(fields): @@ -151,9 +153,27 @@ def get_contact(self, contact_key): 404, u"Contact %r not found." % (contact_key,)) return contact + def _get_contacts_from_query(self, query): + try: + [field, value] = query.split('=') + except ValueError: + raise FakeContactsError( + 400, "Query must be of the form 'field=value'") + if field not in self.valid_search_keys: + raise FakeContactsError( + 400, "Query field must be one of: %s" % self.valid_search_keys) + + contacts = [ + contact for contact in self.contacts_data if + contact[field] == value] + if not contacts: + raise FakeContactsError( + 400, 'Contact with %s %s not found' % (field, value)) + return contacts + def get_all_contacts(self, query): if query is not None: - raise FakeContactsError(400, "query parameter not supported") + raise FakeContactsError(400, 'query parameter not supported') return self.contacts_data.values() def get_all(self, query): @@ -171,6 +191,9 @@ def get_all(self, query): return self.get_page_contacts(q, cursor, max_results) def get_page_contacts(self, query, cursor, max_results): + if query is not None: + return { + u'cursor': None, u'data': self._get_contacts_from_query(query)} contacts = self.get_all_contacts(query) max_results = (max_results and int(max_results)) or float('inf') From bc4cb98a31f5b893e67118952114f23877264ceb Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Fri, 6 Feb 2015 13:15:29 +0200 Subject: [PATCH 06/17] Fix doc strings. --- go_contacts/backends/contacts.py | 5 +++-- verified-fake/fake_go_contacts.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go_contacts/backends/contacts.py b/go_contacts/backends/contacts.py index 0767656..4870b79 100644 --- a/go_contacts/backends/contacts.py +++ b/go_contacts/backends/contacts.py @@ -135,7 +135,8 @@ def stream(self, query): :param unicode query: Search term requested through the API. Defaults to ``None`` if no - search term was requested. Query must be of the form `field=value`. + search term was requested. Currently not implemented and will raise + a CollectionUsageError if not ``None``. """ if query is not None: raise CollectionUsageError("query parameter not supported") @@ -171,7 +172,7 @@ def page(self, cursor, max_results, query): to ``None`` if no limit was specified. :param unicode query: Search term requested through the API. Defaults to ``None`` if no - search term was requested. + search term was requested. Query must be of the form `field=value`. :return: (cursor, data). ``cursor`` is an opaque string that refers to the diff --git a/verified-fake/fake_go_contacts.py b/verified-fake/fake_go_contacts.py index 3203e1b..590ef2a 100644 --- a/verified-fake/fake_go_contacts.py +++ b/verified-fake/fake_go_contacts.py @@ -173,7 +173,7 @@ def _get_contacts_from_query(self, query): def get_all_contacts(self, query): if query is not None: - raise FakeContactsError(400, 'query parameter not supported') + raise FakeContactsError(400, "query parameter not supported") return self.contacts_data.values() def get_all(self, query): From 9b0cc8dcfeef70383066b4d17e5dad397b661f1c Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Fri, 6 Feb 2015 14:34:34 +0200 Subject: [PATCH 07/17] Fix verified fake --- verified-fake/fake_go_contacts.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/verified-fake/fake_go_contacts.py b/verified-fake/fake_go_contacts.py index 590ef2a..eb5cd54 100644 --- a/verified-fake/fake_go_contacts.py +++ b/verified-fake/fake_go_contacts.py @@ -162,13 +162,13 @@ def _get_contacts_from_query(self, query): if field not in self.valid_search_keys: raise FakeContactsError( 400, "Query field must be one of: %s" % self.valid_search_keys) - contacts = [ - contact for contact in self.contacts_data if + contact for key, contact in self.contacts_data.iteritems() if contact[field] == value] if not contacts: raise FakeContactsError( - 400, 'Contact with %s %s not found' % (field, value)) + 400, + "Object u'Contact with %s %s' not found." % (field, value)) return contacts def get_all_contacts(self, query): @@ -479,7 +479,7 @@ def handle_request(self, request): self.build_response("", 404) try: - query_string = parse_qs(urllib.unquote(url.query).decode('utf8')) + query_string = parse_qs(url.query.decode('utf8')) return self.build_response( handler.request( request, contact_key, query_string, self.contacts)) From b989f62954f6cebff83efddcb8995157d3570ac1 Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Fri, 6 Feb 2015 14:37:09 +0200 Subject: [PATCH 08/17] Add test for no contacts found in search. --- .../tests/server_contacts_test_mixin.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/go_contacts/tests/server_contacts_test_mixin.py b/go_contacts/tests/server_contacts_test_mixin.py index 054ae37..ba35746 100644 --- a/go_contacts/tests/server_contacts_test_mixin.py +++ b/go_contacts/tests/server_contacts_test_mixin.py @@ -442,9 +442,26 @@ def test_page_with_query(self): """ api = self.mk_api() contact = yield self.create_contact(api, name=u'Bob', msisdn=u'+12345') + yield self.create_contact(api, name=u'Sue', msisdn=u'+54321') code, data = yield self.request( - api, 'GET', '/contacts/?q="msisdn=+12345"') + api, 'GET', '/contacts/?query=msisdn=%2B12345') self.assertEqual(code, 200) self.assertEqual(data.get(u'cursor'), None) self.assertEqual(data.get('data'), [contact]) + + @inlineCallbacks + def test_page_with_query_no_contact_found(self): + """ + If no contact exists that fulfills the query, a ContactNotFoundError + should be thrown. + """ + api = self.mk_api() + + code, data = yield self.request( + api, 'GET', '/contacts/?query=msisdn=bar') + self.assertEqual(code, 400) + self.assertEqual(data.get(u'status_code'), 400) + self.assertEqual( + data.get('reason'), + u"Object u'Contact with msisdn bar' not found.") From f3625602bca6b86e51cdfc7e174d1426f6ab1f3d Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Fri, 6 Feb 2015 14:38:11 +0200 Subject: [PATCH 09/17] Fix contacts API --- go_contacts/backends/contacts.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/go_contacts/backends/contacts.py b/go_contacts/backends/contacts.py index 4870b79..2d427b2 100644 --- a/go_contacts/backends/contacts.py +++ b/go_contacts/backends/contacts.py @@ -112,19 +112,14 @@ def _get_contacts_by_query(self, query): raise CollectionUsageError( "Query field must be one of: %s" % valid_keys) - contacts = [] - for cls in self.delivery_classes[field]: - try: - contact = ( - yield self.contact_store.contacts.contact_for_addr( - cls, value, create=False)) - contacts.append(contact) - except ContactNotFoundError: - pass - if not contacts: - raise ContactNotFoundError( - 'Contact with %s %s not found' % (field, value)) - returnValue(contacts) + try: + contact = yield self.contact_store.contact_for_addr( + self.delivery_classes[field][0], value, create=False) + except ContactNotFoundError: + raise CollectionObjectNotFound( + 'Contact with %s %s' % (field, value)) + + returnValue([contact_to_dict(contact)]) def stream(self, query): """ From 5d28b4fc8ba037efd613d4db6aab2b4f1e6d8758 Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Fri, 6 Feb 2015 15:15:05 +0200 Subject: [PATCH 10/17] Update travis to install riak 1.4 --- .travis.yml | 17 ++- utils/app.config | 345 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 359 insertions(+), 3 deletions(-) create mode 100644 utils/app.config diff --git a/.travis.yml b/.travis.yml index 3639622..5b3d70e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,10 +2,18 @@ language: python python: - "2.6" - "2.7" -services: - - riak before_install: - # we need the protobuf-compiler so we can install Riak client libraries + # We need Riak 1.4, so we add Basho's repo and install from there. + # Additionally, we remove existing Riak data and replace the config with ours. + - sudo service riak stop + - "curl http://apt.basho.com/gpg/basho.apt.key | sudo apt-key add -" + - sudo bash -c "echo deb http://apt.basho.com $(lsb_release -sc) main > /etc/apt/sources.list.d/basho.list" + - sudo apt-get -qq update + - sudo apt-get install -qq -y --force-yes riak=1.4.12-1 + - sudo rm -rf /var/lib/riak/* + - sudo cp utils/app.config /etc/riak/app.config + - sudo service riak start + # We need the protobuf-compiler so we can install Riak client libraries. - sudo apt-get install -qq protobuf-compiler install: - "pip install -r requirements.txt --use-wheel" @@ -13,6 +21,9 @@ install: - "pip install -e ." # We need to install the verified fake as well so we can test it. - "pip install -e ./verified-fake" + # To see what version of Riak we're running and check that it's happy. + - riak version + - riak-admin member-status script: - coverage run --source=go_contacts `which trial` go_contacts after_success: diff --git a/utils/app.config b/utils/app.config new file mode 100644 index 0000000..90266fc --- /dev/null +++ b/utils/app.config @@ -0,0 +1,345 @@ +%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ft=erlang ts=4 sw=4 et +[ + %% Riak Client APIs config + {riak_api, [ + %% pb_backlog is the maximum length to which the queue of pending + %% connections may grow. If set, it must be an integer >= 0. + %% By default the value is 5. If you anticipate a huge number of + %% connections being initialised *simultaneously*, set this number + %% higher. + %% {pb_backlog, 64}, + + %% pb is a list of IP addresses and TCP ports that the Riak + %% Protocol Buffers interface will bind. + {pb, [ {"127.0.0.1", 8087 } ]} + ]}, + + %% Riak Core config + {riak_core, [ + %% Default location of ringstate + {ring_state_dir, "/var/lib/riak/ring"}, + + %% Default ring creation size. Make sure it is a power of 2, + %% e.g. 16, 32, 64, 128, 256, 512 etc + %{ring_creation_size, 64}, + {ring_creation_size, 4}, + + %% http is a list of IP addresses and TCP ports that the Riak + %% HTTP interface will bind. + {http, [ {"127.0.0.1", 8098 } ]}, + + %% https is a list of IP addresses and TCP ports that the Riak + %% HTTPS interface will bind. + %{https, [{ "127.0.0.1", 8098 }]}, + + %% Default cert and key locations for https can be overridden + %% with the ssl config variable, for example: + %{ssl, [ + % {certfile, "/etc/riak/cert.pem"}, + % {keyfile, "/etc/riak/key.pem"} + % ]}, + + %% riak_handoff_port is the TCP port that Riak uses for + %% intra-cluster data handoff. + {handoff_port, 8099 }, + + %% To encrypt riak_core intra-cluster data handoff traffic, + %% uncomment the following line and edit its path to an + %% appropriate certfile and keyfile. (This example uses a + %% single file with both items concatenated together.) + %{handoff_ssl_options, [{certfile, "/tmp/erlserver.pem"}]}, + + %% DTrace support + %% Do not enable 'dtrace_support' unless your Erlang/OTP + %% runtime is compiled to support DTrace. DTrace is + %% available in R15B01 (supported by the Erlang/OTP + %% official source package) and in R14B04 via a custom + %% source repository & branch. + {dtrace_support, false}, + + %% Health Checks + %% If disabled, health checks registered by an application will + %% be ignored. NOTE: this option cannot be changed at runtime. + %% To re-enable, the setting must be changed and the node restarted. + %% NOTE: As of Riak 1.3.2, health checks are deprecated as they + %% may interfere with the new overload protection mechanisms. + %% If there is a good reason to re-enable them, you must uncomment + %% this line and also add an entry in the riak_kv section: + %% {riak_kv, [ ..., {enable_health_checks, true}, ...]} + %% {enable_health_checks, true}, + + %% Platform-specific installation paths (substituted by rebar) + {platform_bin_dir, "/usr/sbin"}, + {platform_data_dir, "/var/lib/riak"}, + {platform_etc_dir, "/etc/riak"}, + {platform_lib_dir, "/usr/lib/riak/lib"}, + {platform_log_dir, "/var/log/riak"} + ]}, + + %% Riak KV config + {riak_kv, [ + %% Storage_backend specifies the Erlang module defining the storage + %% mechanism that will be used on this node. + {storage_backend, riak_kv_eleveldb_backend}, + + %% raw_name is the first part of all URLS used by the Riak raw HTTP + %% interface. See riak_web.erl and raw_http_resource.erl for + %% details. + %{raw_name, "riak"}, + + %% Enable active anti-entropy subsystem + optional debug messages: + %% {anti_entropy, {on|off, []}}, + %% {anti_entropy, {on|off, [debug]}}, + {anti_entropy, {off, []}}, + + %% Restrict how fast AAE can build hash trees. Building the tree + %% for a given partition requires a full scan over that partition's + %% data. Once built, trees stay built until they are expired. + %% Config is of the form: + %% {num-builds, per-timespan-in-milliseconds} + %% Default is 1 build per hour. + {anti_entropy_build_limit, {1, 3600000}}, + + %% Determine how often hash trees are expired after being built. + %% Periodically expiring a hash tree ensures the on-disk hash tree + %% data stays consistent with the actual k/v backend data. It also + %% helps Riak identify silent disk failures and bit rot. However, + %% expiration is not needed for normal AAE operation and should be + %% infrequent for performance reasons. The time is specified in + %% milliseconds. The default is 1 week. + {anti_entropy_expire, 604800000}, + + %% Limit how many AAE exchanges/builds can happen concurrently. + {anti_entropy_concurrency, 2}, + + %% The tick determines how often the AAE manager looks for work + %% to do (building/expiring trees, triggering exchanges, etc). + %% The default is every 15 seconds. Lowering this value will + %% speedup the rate that all replicas are synced across the cluster. + %% Increasing the value is not recommended. + {anti_entropy_tick, 15000}, + + %% The directory where AAE hash trees are stored. + {anti_entropy_data_dir, "/var/lib/riak/anti_entropy"}, + + %% The LevelDB options used by AAE to generate the LevelDB-backed + %% on-disk hashtrees. + {anti_entropy_leveldb_opts, [{write_buffer_size, 4194304}, + {max_open_files, 20}]}, + + %% mapred_name is URL used to submit map/reduce requests to Riak. + {mapred_name, "mapred"}, + + %% mapred_2i_pipe indicates whether secondary-index + %% MapReduce inputs are queued in parallel via their own + %% pipe ('true'), or serially via a helper process + %% ('false' or undefined). Set to 'false' or leave + %% undefined during a rolling upgrade from 1.0. + {mapred_2i_pipe, true}, + + %% Each of the following entries control how many Javascript + %% virtual machines are available for executing map, reduce, + %% pre- and post-commit hook functions. + {map_js_vm_count, 8 }, + {reduce_js_vm_count, 6 }, + {hook_js_vm_count, 2 }, + + %% js_max_vm_mem is the maximum amount of memory, in megabytes, + %% allocated to the Javascript VMs. If unset, the default is + %% 8MB. + {js_max_vm_mem, 8}, + + %% js_thread_stack is the maximum amount of thread stack, in megabyes, + %% allocate to the Javascript VMs. If unset, the default is 16MB. + %% NOTE: This is not the same as the C thread stack. + {js_thread_stack, 16}, + + %% js_source_dir should point to a directory containing Javascript + %% source files which will be loaded by Riak when it initializes + %% Javascript VMs. + %{js_source_dir, "/tmp/js_source"}, + + %% http_url_encoding determines how Riak treats URL encoded + %% buckets, keys, and links over the REST API. When set to 'on' + %% Riak always decodes encoded values sent as URLs and Headers. + %% Otherwise, Riak defaults to compatibility mode where links + %% are decoded, but buckets and keys are not. The compatibility + %% mode will be removed in a future release. + {http_url_encoding, on}, + + %% Switch to vnode-based vclocks rather than client ids. This + %% significantly reduces the number of vclock entries. + %% Only set true if *all* nodes in the cluster are upgraded to 1.0 + {vnode_vclocks, true}, + + %% This option toggles compatibility of keylisting with 1.0 + %% and earlier versions. Once a rolling upgrade to a version + %% > 1.0 is completed for a cluster, this should be set to + %% true for better control of memory usage during key listing + %% operations + {listkeys_backpressure, true}, + + %% This option specifies how many of each type of fsm may exist + %% concurrently. This is for overload protection and is a new + %% mechanism that obsoletes 1.3's health checks. Note that this number + %% represents two potential processes, so +P in vm.args should be at + %% least 3X the fsm_limit. + {fsm_limit, 50000}, + + %% Uncomment to make non-paginated results be sorted the + %% same way paginated results are: by term, then key. + %% In Riak 1.4.* before 1.4.4, all results were sorted this way + %% by default, which can adversely affect performance in some cases. + %% Setting this to true emulates that behavior. + %% {secondary_index_sort_default, true}, + + %% object_format controls which binary representation of a riak_object + %% is stored on disk. + %% Current options are: v0, v1. + %% v0: Original erlang:term_to_binary format. Higher space overhead. + %% v1: New format for more compact storage of small values. + {object_format, v1} + ]}, + + %% Riak Search Config + {riak_search, [ + %% To enable Search functionality set this 'true'. + {enabled, true} + ]}, + + %% Merge Index Config + {merge_index, [ + %% The root dir to store search merge_index data + {data_root, "/var/lib/riak/merge_index"}, + + %% Size, in bytes, of the in-memory buffer. When this + %% threshold has been reached the data is transformed + %% into a segment file which resides on disk. + {buffer_rollover_size, 1048576}, + + %% Overtime the segment files need to be compacted. + %% This is the maximum number of segments that will be + %% compacted at once. A lower value will lead to + %% quicker but more frequent compactions. + {max_compact_segments, 20} + ]}, + + %% Bitcask Config + {bitcask, [ + %% Configure how Bitcask writes data to disk. + %% erlang: Erlang's built-in file API + %% nif: Direct calls to the POSIX C API + %% + %% The NIF mode provides higher throughput for certain + %% workloads, but has the potential to negatively impact + %% the Erlang VM, leading to higher worst-case latencies + %% and possible throughput collapse. + {io_mode, erlang}, + + {data_root, "/var/lib/riak/bitcask"} + ]}, + + %% eLevelDB Config + {eleveldb, [ + {data_root, "/var/lib/riak/leveldb"} + ]}, + + %% Lager Config + {lager, [ + %% What handlers to install with what arguments + %% The defaults for the logfiles are to rotate the files when + %% they reach 10Mb or at midnight, whichever comes first, and keep + %% the last 5 rotations. See the lager README for a description of + %% the time rotation format: + %% https://github.com/basho/lager/blob/master/README.org + %% + %% If you wish to disable rotation, you can either set the size to 0 + %% and the rotation time to "", or instead specify a 2-tuple that only + %% consists of {Logfile, Level}. + %% + %% If you wish to have riak log messages to syslog, you can use a handler + %% like this: + %% {lager_syslog_backend, ["riak", daemon, info]}, + %% + {handlers, [ + {lager_file_backend, [ + {"/var/log/riak/error.log", error, 10485760, "$D0", 5}, + {"/var/log/riak/console.log", info, 10485760, "$D0", 5} + ]} + ] }, + + %% Whether to write a crash log, and where. + %% Commented/omitted/undefined means no crash logger. + {crash_log, "/var/log/riak/crash.log"}, + + %% Maximum size in bytes of events in the crash log - defaults to 65536 + {crash_log_msg_size, 65536}, + + %% Maximum size of the crash log in bytes, before its rotated, set + %% to 0 to disable rotation - default is 0 + {crash_log_size, 10485760}, + + %% What time to rotate the crash log - default is no time + %% rotation. See the lager README for a description of this format: + %% https://github.com/basho/lager/blob/master/README.org + {crash_log_date, "$D0"}, + + %% Number of rotated crash logs to keep, 0 means keep only the + %% current one - default is 0 + {crash_log_count, 5}, + + %% Whether to redirect error_logger messages into lager - defaults to true + {error_logger_redirect, true}, + + %% maximum number of error_logger messages to handle in a second + %% lager 2.0.0 shipped with a limit of 50, which is a little low for riak's startup + {error_logger_hwm, 100} + ]}, + + %% riak_sysmon config + {riak_sysmon, [ + %% To disable forwarding events of a particular type, use a + %% limit of 0. + {process_limit, 30}, + {port_limit, 2}, + + %% Finding reasonable limits for a given workload is a matter + %% of experimentation. + %% NOTE: Enabling the 'gc_ms_limit' monitor (by setting non-zero) + %% can cause performance problems on multi-CPU systems. + {gc_ms_limit, 0}, + {heap_word_limit, 40111000}, + + %% Configure the following items to 'false' to disable logging + %% of that event type. + {busy_port, true}, + {busy_dist_port, true} + ]}, + + %% SASL config + {sasl, [ + {sasl_error_logger, false} + ]}, + + %% riak_control config + {riak_control, [ + %% Set to false to disable the admin panel. + {enabled, false}, + + %% Authentication style used for access to the admin + %% panel. Valid styles are 'userlist' . + {auth, userlist}, + + %% If auth is set to 'userlist' then this is the + %% list of usernames and passwords for access to the + %% admin panel. + {userlist, [{"user", "pass"} + ]}, + + %% The admin panel is broken up into multiple + %% components, each of which is enabled or disabled + %% by one of these settings. + {admin, true} + ]} +]. \ No newline at end of file From 1a190353c337cfc3dec79865684fc467f666d530 Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Fri, 6 Feb 2015 16:46:40 +0200 Subject: [PATCH 11/17] Sort list of address types for error. --- go_contacts/backends/contacts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go_contacts/backends/contacts.py b/go_contacts/backends/contacts.py index 2d427b2..fd50a21 100644 --- a/go_contacts/backends/contacts.py +++ b/go_contacts/backends/contacts.py @@ -107,10 +107,10 @@ def _get_contacts_by_query(self, query): except ValueError: raise CollectionUsageError( "Query must be of the form 'field=value'") - valid_keys = self.delivery_classes.keys() - if field not in valid_keys: + if field not in self.delivery_classes: raise CollectionUsageError( - "Query field must be one of: %s" % valid_keys) + "Query field must be one of: %s" % + sorted(self.delivery_classes.keys())) try: contact = yield self.contact_store.contact_for_addr( From 651f10f2946b748cc841a811d6c491da744bae36 Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Fri, 6 Feb 2015 16:58:47 +0200 Subject: [PATCH 12/17] Update tests. --- go_contacts/tests/server_contacts_test_mixin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go_contacts/tests/server_contacts_test_mixin.py b/go_contacts/tests/server_contacts_test_mixin.py index ba35746..263b6b2 100644 --- a/go_contacts/tests/server_contacts_test_mixin.py +++ b/go_contacts/tests/server_contacts_test_mixin.py @@ -432,8 +432,8 @@ def test_page_invalid_query_parameter(self): self.assertEqual(data.get(u'status_code'), 400) self.assertEqual( data.get(u'reason'), - u"Query field must be one of: ['msisdn', 'wechat_id', 'gtalk_id'," - " 'twitter_handle', 'mxit_id']") + u"Query field must be one of: ['gtalk_id', 'msisdn', 'mxit_id', " + "'twitter_handle', 'wechat_id']") @inlineCallbacks def test_page_with_query(self): From b955e4d07386782a38958321320f065819cd7d0f Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Fri, 6 Feb 2015 16:59:00 +0200 Subject: [PATCH 13/17] Sort fields in FakeContactsApi error message. --- verified-fake/fake_go_contacts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/verified-fake/fake_go_contacts.py b/verified-fake/fake_go_contacts.py index eb5cd54..83eaa1a 100644 --- a/verified-fake/fake_go_contacts.py +++ b/verified-fake/fake_go_contacts.py @@ -161,7 +161,8 @@ def _get_contacts_from_query(self, query): 400, "Query must be of the form 'field=value'") if field not in self.valid_search_keys: raise FakeContactsError( - 400, "Query field must be one of: %s" % self.valid_search_keys) + 400, "Query field must be one of: %s" + % sorted(self.valid_search_keys)) contacts = [ contact for key, contact in self.contacts_data.iteritems() if contact[field] == value] From 86c05896fc8654cd9024b0a6d929553197ee0d1e Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Mon, 9 Feb 2015 10:23:12 +0200 Subject: [PATCH 14/17] Update contacts API to use new vumi-go find by addr field function. --- go_contacts/backends/contacts.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/go_contacts/backends/contacts.py b/go_contacts/backends/contacts.py index fd50a21..b9510e5 100644 --- a/go_contacts/backends/contacts.py +++ b/go_contacts/backends/contacts.py @@ -1,7 +1,6 @@ """ Riak contacts backend and collection. """ -from collections import defaultdict from twisted.internet.defer import inlineCallbacks, returnValue from zope.interface import implementer @@ -10,7 +9,7 @@ from go.vumitools.contact import ( ContactStore, ContactNotFoundError, Contact) -from go.vumitools.contact.models import DELIVERY_CLASSES +from go.vumitools.contact.models import normalize_addr from go_api.collections import ICollection from go_api.collections.errors import ( @@ -54,7 +53,6 @@ class RiakContactsCollection(object): def __init__(self, contact_store, max_contacts_per_page): self.contact_store = contact_store self.max_contacts_per_page = max_contacts_per_page - self.delivery_classes = self._reverse_delivery_class() @staticmethod def _pick_fields(data, keys): @@ -94,12 +92,6 @@ def all_keys(self): """ raise NotImplementedError() - def _reverse_delivery_class(self): - res = defaultdict(list) - for cls, value in DELIVERY_CLASSES.iteritems(): - res[value['field']].append(cls) - return res - @inlineCallbacks def _get_contacts_by_query(self, query): try: @@ -107,14 +99,15 @@ def _get_contacts_by_query(self, query): except ValueError: raise CollectionUsageError( "Query must be of the form 'field=value'") - if field not in self.delivery_classes: + if field not in Contact.ADDRESS_FIELDS: raise CollectionUsageError( "Query field must be one of: %s" % - sorted(self.delivery_classes.keys())) + sorted(Contact.ADDRESS_FIELDS)) try: - contact = yield self.contact_store.contact_for_addr( - self.delivery_classes[field][0], value, create=False) + value = normalize_addr(field, value) + contact = yield self.contact_store.contact_for_addr_field( + field, value, create=False) except ContactNotFoundError: raise CollectionObjectNotFound( 'Contact with %s %s' % (field, value)) From 5ea208275cf618384a8324aadc5e4a10c6824a01 Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Mon, 9 Feb 2015 10:23:35 +0200 Subject: [PATCH 15/17] Update tests. --- go_contacts/tests/server_contacts_test_mixin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go_contacts/tests/server_contacts_test_mixin.py b/go_contacts/tests/server_contacts_test_mixin.py index 263b6b2..31e30ac 100644 --- a/go_contacts/tests/server_contacts_test_mixin.py +++ b/go_contacts/tests/server_contacts_test_mixin.py @@ -432,8 +432,8 @@ def test_page_invalid_query_parameter(self): self.assertEqual(data.get(u'status_code'), 400) self.assertEqual( data.get(u'reason'), - u"Query field must be one of: ['gtalk_id', 'msisdn', 'mxit_id', " - "'twitter_handle', 'wechat_id']") + u"Query field must be one of: ['bbm_pin', 'facebook_id', " + "'gtalk_id', 'msisdn', 'mxit_id', 'twitter_handle', 'wechat_id']") @inlineCallbacks def test_page_with_query(self): @@ -464,4 +464,4 @@ def test_page_with_query_no_contact_found(self): self.assertEqual(data.get(u'status_code'), 400) self.assertEqual( data.get('reason'), - u"Object u'Contact with msisdn bar' not found.") + u"Object u'Contact with msisdn +bar' not found.") From 1cc893fbaf3d00f18afb3f9c94a8b8f79f5a14f9 Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Mon, 9 Feb 2015 10:23:44 +0200 Subject: [PATCH 16/17] Update verified fake. --- verified-fake/fake_go_contacts.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/verified-fake/fake_go_contacts.py b/verified-fake/fake_go_contacts.py index 83eaa1a..45ca0ec 100644 --- a/verified-fake/fake_go_contacts.py +++ b/verified-fake/fake_go_contacts.py @@ -96,7 +96,8 @@ def __init__(self, contacts_data={}, max_contacts_per_page=10): self.contacts_data = contacts_data self.max_contacts_per_page = max_contacts_per_page self.valid_search_keys = [ - 'msisdn', 'wechat_id', 'gtalk_id', 'twitter_handle', 'mxit_id'] + 'bbm_pin', 'facebook_id', 'gtalk_id', 'msisdn', 'mxit_id', + 'twitter_handle', 'wechat_id'] @staticmethod def make_contact_dict(fields): @@ -153,6 +154,13 @@ def get_contact(self, contact_key): 404, u"Contact %r not found." % (contact_key,)) return contact + def _normalize_addr(self, contact_field, addr): + if contact_field == 'msisdn': + addr = '+' + addr.lstrip('+') + elif contact_field == 'gtalk': + addr = addr.partition('/')[0] + return addr + def _get_contacts_from_query(self, query): try: [field, value] = query.split('=') @@ -161,8 +169,9 @@ def _get_contacts_from_query(self, query): 400, "Query must be of the form 'field=value'") if field not in self.valid_search_keys: raise FakeContactsError( - 400, "Query field must be one of: %s" + 400, "Query field must be one of: %s" % sorted(self.valid_search_keys)) + value = self._normalize_addr(field, value) contacts = [ contact for key, contact in self.contacts_data.iteritems() if contact[field] == value] From 67f5903d65b05ff7353cae847fba3a4c57aeff10 Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Mon, 9 Feb 2015 12:51:05 +0200 Subject: [PATCH 17/17] Move normalize_addr call to outside of try statement. --- go_contacts/backends/contacts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go_contacts/backends/contacts.py b/go_contacts/backends/contacts.py index b9510e5..0aaab07 100644 --- a/go_contacts/backends/contacts.py +++ b/go_contacts/backends/contacts.py @@ -104,8 +104,8 @@ def _get_contacts_by_query(self, query): "Query field must be one of: %s" % sorted(Contact.ADDRESS_FIELDS)) + value = normalize_addr(field, value) try: - value = normalize_addr(field, value) contact = yield self.contact_store.contact_for_addr_field( field, value, create=False) except ContactNotFoundError: