From 9373edf9c71dcbaaa4ed9beb9fda52a925ea1301 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Wed, 19 Aug 2015 16:39:19 +0200 Subject: [PATCH 001/111] Tag services with the registration ID and clean old services * Tag services with Consular's registration ID so that the services are "owned" by Consular and can be trimmed when they no longer exist. --- consular/cli.py | 7 +++- consular/main.py | 81 +++++++++++++++++++++++++------------ consular/tests/test_main.py | 77 +++++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 27 deletions(-) diff --git a/consular/cli.py b/consular/cli.py index 21dbaa9..8b29cad 100644 --- a/consular/cli.py +++ b/consular/cli.py @@ -17,8 +17,10 @@ help='The Marathon HTTP API') @click.option('--registration-id', help=('Auto register for Marathon event callbacks with the ' - 'registration-id. Must be unique for each consular ' - 'process.'), type=str) + 'registration-id. Also used to identify which services in ' + 'Consul should be maintained by consular. Must be unique ' + 'for each consular process.'), + type=str) @click.option('--sync-interval', help=('Automatically sync the apps in Marathon with what\'s ' 'in Consul every _n_ seconds. Defaults to 0 (disabled).'), @@ -64,6 +66,7 @@ def main(scheme, host, port, consular.timeout = timeout consular.fallback_timeout = fallback_timeout if registration_id: + consular.registration_id = registration_id events_url = "%s://%s:%s/events?%s" % ( scheme, host, port, urlencode({ diff --git a/consular/main.py b/consular/main.py index 67b7a19..cdf3cd4 100644 --- a/consular/main.py +++ b/consular/main.py @@ -38,6 +38,7 @@ class Consular(object): clock = reactor timeout = 5 fallback_timeout = 2 + registration_id = None requester = lambda self, *a, **kw: treq.request(*a, **kw) def __init__(self, consul_endpoint, marathon_endpoint, enable_fallback): @@ -182,38 +183,49 @@ def handle_unknown_event(self, request, event): 'error': 'Event type %s not supported.' % (event_type,) }) + def _registration_tag(self): + """ + Get the Consul service tag used to mark a service as created by this + instance of Consular. + """ + return 'consular-reg-id:%s' % (self.registration_id,) + + def _create_service_registration(self, app_id, service_id, address, port): + """ + Create the request body for registering a service with Consul. + """ + registration = { + 'Name': app_id, + 'ID': service_id, + 'Address': address, + 'Port': port + } + if self.registration_id: + registration['Tags'] = [self._registration_tag()] + return registration + def register_service(self, agent_endpoint, app_id, service_id, address, port): log.msg('Registering %s at %s with %s at %s:%s.' % ( app_id, agent_endpoint, service_id, address, port)) + registration = self._create_service_registration(app_id, service_id, + address, port) + d = self.consul_request( 'PUT', '%s/v1/agent/service/register' % (agent_endpoint,), - { - 'Name': app_id, - 'ID': service_id, - 'Address': address, - 'Port': port, - }) + registration) if self.enable_fallback: - d.addErrback( - self.register_service_fallback, app_id, service_id, - address, port) + d.addErrback(self.register_service_fallback, registration) return d - def register_service_fallback(self, failure, - app_id, service_id, address, port): - log.msg('Falling back for %s at %s with %s at %s:%s.' % ( - app_id, self.consul_endpoint, service_id, address, port)) + def register_service_fallback(self, failure, registration): + log.msg('Falling back for %s at %s.' % ( + registration['Name'], self.consul_endpoint)) return self.consul_request( 'PUT', '%s/v1/agent/service/register' % (self.consul_endpoint,), - { - 'Name': app_id, - 'ID': service_id, - 'Address': address, - 'Port': port, - }) + registration) def deregister_service(self, agent_endpoint, app_id, service_id): log.msg('Deregistering %s at %s with %s' % ( @@ -288,22 +300,41 @@ def purge_dead_agent_services(self, agent_endpoint): # collect the task ids for the service name services = {} for service_id, service in data.items(): - services.setdefault(service['Service'], set([])).add(service_id) + # If we have a registration ID, check the service for a tag that + # matches our registration ID + if (not self.registration_id + or self._is_registration_in_tags(service['Tags'])): + services.setdefault(service['Service'], set([])).add( + service_id) for app_id, task_ids in services.items(): yield self.purge_service_if_dead(agent_endpoint, app_id, task_ids) + def _is_registration_in_tags(self, tags): + """ + Check if the Consul service was tagged with our registration ID. + """ + if not tags: + return False + + return self._registration_tag() in tags + @inlineCallbacks def purge_service_if_dead(self, agent_endpoint, app_id, consul_task_ids): response = yield self.marathon_request( 'GET', '/v2/apps/%s/tasks' % (app_id,)) data = yield response.json() + tasks_to_be_purged = set(consul_task_ids) if 'tasks' not in data: - log.msg(('App %s does not look like a Marathon application, ' - 'skipping') % (str(app_id),)) - return + # If there are no matching tasks in Marathon and we haven't matched + # the service by registration ID, then skip it. + if not self.registration_id: + log.msg(('App %s does not look like a Marathon application, ' + 'skipping') % (str(app_id),)) + return + else: + marathon_task_ids = set([task['id'] for task in data['tasks']]) + tasks_to_be_purged -= marathon_task_ids - marathon_task_ids = set([task['id'] for task in data['tasks']]) - tasks_to_be_purged = consul_task_ids - marathon_task_ids for task_id in tasks_to_be_purged: yield self.deregister_service(agent_endpoint, app_id, task_id) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 70e81b1..a6b3f9f 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -398,6 +398,83 @@ def test_purge_dead_services(self): FakeResponse(200, [], json.dumps({}))) yield d + @inlineCallbacks + def test_purge_old_services(self): + """ + Services previously registered with Consul by Consular but that no + longer exist in Marathon should be purged if a registration ID is set. + """ + self.consular.registration_id = "test" + d = self.consular.purge_dead_services() + consul_request = yield self.requests.get() + self.assertEqual( + consul_request['url'], + 'http://localhost:8500/v1/catalog/nodes') + consul_request['deferred'].callback( + FakeResponse(200, [], json.dumps([{ + 'Node': 'consul-node', + 'Address': '1.2.3.4', + }])) + ) + agent_request = yield self.requests.get() + # Expecting a request to list of all services in Consul, returning 3 + # services - one tagged with our registration ID, one tagged with a + # different registration ID, and one with no tags. + self.assertEqual( + agent_request['url'], + 'http://1.2.3.4:8500/v1/agent/services') + self.assertEqual(agent_request['method'], 'GET') + agent_request['deferred'].callback( + FakeResponse(200, [], json.dumps({ + "testingapp.someid1": { + "ID": "testingapp.someid1", + "Service": "testingapp", + "Tags": [ + "consular-reg-id:test" + ], + "Address": "machine-1", + "Port": 8102 + }, + "testingapp.someid2": { + "ID": "testingapp.someid2", + "Service": "testingapp", + "Tags": [ + "consular-reg-id:blah" + ], + "Address": "machine-2", + "Port": 8103 + }, + "testingapp.someid3": { + "ID": "testingapp.someid2", + "Service": "testingapp", + "Tags": None, + "Address": "machine-2", + "Port": 8104 + } + })) + ) + + # Expecting a request for the tasks for a given app, returning no tasks + testingapp_request = yield self.requests.get() + self.assertEqual(testingapp_request['url'], + 'http://localhost:8080/v2/apps/testingapp/tasks') + self.assertEqual(testingapp_request['method'], 'GET') + testingapp_request['deferred'].callback( + FakeResponse(200, [], json.dumps({})) + ) + + # Expecting a service deregistering in Consul as a result. Only the + # task with the correct tag is returned. + deregister_request = yield self.requests.get() + self.assertEqual( + deregister_request['url'], + ('http://1.2.3.4:8500/v1/agent/service/deregister/' + 'testingapp.someid1')) + self.assertEqual(deregister_request['method'], 'PUT') + deregister_request['deferred'].callback( + FakeResponse(200, [], json.dumps({}))) + yield d + @inlineCallbacks def test_fallback_to_main_consul(self): self.consular.enable_fallback = True From 158e10796d123a27208de55e1ddaa8ca0b2dbfbd Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Thu, 20 Aug 2015 18:48:22 +0200 Subject: [PATCH 002/111] Log the URL when a request fails in debug mode --- consular/main.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/consular/main.py b/consular/main.py index 67b7a19..0304dc8 100644 --- a/consular/main.py +++ b/consular/main.py @@ -104,6 +104,10 @@ def log_http_response(self, response, method, path, data): method, path, data, response.code)) return response + def log_http_error(self, failure, url): + log.msg('Error performing request to %s' % (url,)) + failure.raiseException() + def marathon_request(self, method, path, data=None): return self._request( method, '%s%s' % (self.marathon_endpoint, path), data) @@ -124,6 +128,7 @@ def _request(self, method, url, data, timeout=None): timeout=timeout or self.timeout) if self.debug: d.addCallback(self.log_http_response, method, url, data) + d.addErrback(self.log_http_error, url) return d @app.route('/') From e00fb9fab19c83af7eb98d5538630c5e3b7aa7eb Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Wed, 26 Aug 2015 15:47:13 +0200 Subject: [PATCH 003/111] Use ls | explode to reduce consul-template logspam --- templates/nginx.ctmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/nginx.ctmpl b/templates/nginx.ctmpl index 88e3f67..078b52a 100644 --- a/templates/nginx.ctmpl +++ b/templates/nginx.ctmpl @@ -7,7 +7,7 @@ # Nginx. If the key does not exist the service isn't added to the # list of services in the Nginx config. -{{range services}}{{if key (print "consular/" .Name "/domain") }} +{{range services}}{{$labels = ls (print "consular/" .Name) | explode }}{{if $labels.domain}} upstream {{.Name}} { {{range service .Name }}server {{.Address}}:{{.Port}}; @@ -16,7 +16,7 @@ upstream {{.Name}} { server { listen 80; - server_name {{key (print "consular/" .Name "/domain") | parseJSON }}; + server_name {{$labels.domain | parseJSON }}; location / { proxy_pass http://{{.Name}}; From 5e10557762c08db6e44cb0113fcf94ad828dc38a Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Thu, 3 Sep 2015 11:27:43 +0200 Subject: [PATCH 004/111] Make --registration-id a required parameter --- consular/cli.py | 16 +++++++--------- consular/main.py | 25 ++++++++----------------- consular/tests/test_main.py | 13 +++++++++---- 3 files changed, 24 insertions(+), 30 deletions(-) diff --git a/consular/cli.py b/consular/cli.py index 8b29cad..3cdb1a7 100644 --- a/consular/cli.py +++ b/consular/cli.py @@ -20,7 +20,7 @@ 'registration-id. Also used to identify which services in ' 'Consul should be maintained by consular. Must be unique ' 'for each consular process.'), - type=str) + type=str, required=True) @click.option('--sync-interval', help=('Automatically sync the apps in Marathon with what\'s ' 'in Consul every _n_ seconds. Defaults to 0 (disabled).'), @@ -65,14 +65,12 @@ def main(scheme, host, port, consular.debug = debug consular.timeout = timeout consular.fallback_timeout = fallback_timeout - if registration_id: - consular.registration_id = registration_id - events_url = "%s://%s:%s/events?%s" % ( - scheme, host, port, - urlencode({ - 'registration': registration_id, - })) - consular.register_marathon_event_callback(events_url) + events_url = "%s://%s:%s/events?%s" % ( + scheme, host, port, + urlencode({ + 'registration': registration_id, + })) + consular.register_marathon_event_callback(events_url) if sync_interval > 0: lc = LoopingCall(consular.sync_apps, purge) diff --git a/consular/main.py b/consular/main.py index cdf3cd4..99c0747 100644 --- a/consular/main.py +++ b/consular/main.py @@ -38,14 +38,15 @@ class Consular(object): clock = reactor timeout = 5 fallback_timeout = 2 - registration_id = None requester = lambda self, *a, **kw: treq.request(*a, **kw) - def __init__(self, consul_endpoint, marathon_endpoint, enable_fallback): + def __init__(self, consul_endpoint, marathon_endpoint, enable_fallback, + registration_id): self.consul_endpoint = consul_endpoint self.marathon_endpoint = marathon_endpoint self.pool = client.HTTPConnectionPool(self.clock, persistent=False) self.enable_fallback = enable_fallback + self.registration_id = registration_id self.event_dispatch = { 'status_update_event': self.handle_status_update_event, } @@ -198,10 +199,9 @@ def _create_service_registration(self, app_id, service_id, address, port): 'Name': app_id, 'ID': service_id, 'Address': address, - 'Port': port + 'Port': port, + 'Tags': [self._registration_tag()] } - if self.registration_id: - registration['Tags'] = [self._registration_tag()] return registration def register_service(self, agent_endpoint, @@ -300,10 +300,8 @@ def purge_dead_agent_services(self, agent_endpoint): # collect the task ids for the service name services = {} for service_id, service in data.items(): - # If we have a registration ID, check the service for a tag that - # matches our registration ID - if (not self.registration_id - or self._is_registration_in_tags(service['Tags'])): + # Check the service for a tag that matches our registration ID + if self._is_registration_in_tags(service['Tags']): services.setdefault(service['Service'], set([])).add( service_id) @@ -325,14 +323,7 @@ def purge_service_if_dead(self, agent_endpoint, app_id, consul_task_ids): 'GET', '/v2/apps/%s/tasks' % (app_id,)) data = yield response.json() tasks_to_be_purged = set(consul_task_ids) - if 'tasks' not in data: - # If there are no matching tasks in Marathon and we haven't matched - # the service by registration ID, then skip it. - if not self.registration_id: - log.msg(('App %s does not look like a Marathon application, ' - 'skipping') % (str(app_id),)) - return - else: + if 'tasks' in data: marathon_task_ids = set([task['id'] for task in data['tasks']]) tasks_to_be_purged -= marathon_task_ids diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index a6b3f9f..455f2e5 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -38,7 +38,8 @@ def setUp(self): self.consular = Consular( 'http://localhost:8500', 'http://localhost:8080', - False + False, + 'test' ) self.consular.debug = True @@ -169,6 +170,7 @@ def test_TASK_RUNNING(self): 'ID': 'my-app_0-1396592784349', 'Address': 'slave-1234.acme.org', 'Port': 31372, + 'Tags': ['consular-reg-id:test'], })) request['deferred'].callback( FakeResponse(200, [], json.dumps({}))) @@ -279,6 +281,7 @@ def test_sync_app_task(self): 'ID': 'my-task-id', 'Address': '0.0.0.0', 'Port': 1234, + 'Tags': ['consular-reg-id:test'], })) self.assertEqual(consul_request['method'], 'PUT') consul_request['deferred'].callback( @@ -354,14 +357,16 @@ def test_purge_dead_services(self): "Service": "testingapp", "Tags": None, "Address": "machine-1", - "Port": 8102 + "Port": 8102, + 'Tags': ['consular-reg-id:test'], }, "testingapp.someid2": { "ID": "testingapp.someid2", "Service": "testingapp", "Tags": None, "Address": "machine-2", - "Port": 8103 + "Port": 8103, + 'Tags': ['consular-reg-id:test'], } })) ) @@ -404,7 +409,6 @@ def test_purge_old_services(self): Services previously registered with Consul by Consular but that no longer exist in Marathon should be purged if a registration ID is set. """ - self.consular.registration_id = "test" d = self.consular.purge_dead_services() consul_request = yield self.requests.get() self.assertEqual( @@ -495,4 +499,5 @@ def test_fallback_to_main_consul(self): 'ID': 'service_id', 'Address': 'foo', 'Port': 1234, + 'Tags': ['consular-reg-id:test'], })) From 40fe6053bfced460859f759d77eb12743e606a99 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Thu, 3 Sep 2015 11:28:58 +0200 Subject: [PATCH 005/111] Add coverage HTML directory to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 70345d0..07458b6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ .coverage _trial_temp/ /docs/_build +htmlcov/ From 4f3768953bccf662990451bf444f2eab39811ce4 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Thu, 3 Sep 2015 11:34:39 +0200 Subject: [PATCH 006/111] Rework registration-id param: default to 'consular', change help message --- consular/cli.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/consular/cli.py b/consular/cli.py index 3cdb1a7..683c86d 100644 --- a/consular/cli.py +++ b/consular/cli.py @@ -16,11 +16,10 @@ @click.option('--marathon', default='http://localhost:8080', help='The Marathon HTTP API') @click.option('--registration-id', - help=('Auto register for Marathon event callbacks with the ' - 'registration-id. Also used to identify which services in ' - 'Consul should be maintained by consular. Must be unique ' + help=('Name used to register for event callbacks in Marathon as ' + 'well as to register services in Consul. Must be unique ' 'for each consular process.'), - type=str, required=True) + type=str, default='consular') @click.option('--sync-interval', help=('Automatically sync the apps in Marathon with what\'s ' 'in Consul every _n_ seconds. Defaults to 0 (disabled).'), From 4ae73970d76c616370af827dda9547263f622801 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Thu, 3 Sep 2015 11:46:48 +0200 Subject: [PATCH 007/111] Actually pass the registration-id value --- consular/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consular/cli.py b/consular/cli.py index 683c86d..08406ca 100644 --- a/consular/cli.py +++ b/consular/cli.py @@ -60,7 +60,7 @@ def main(scheme, host, port, log.startLogging(logfile) - consular = Consular(consul, marathon, fallback) + consular = Consular(consul, marathon, fallback, registration_id) consular.debug = debug consular.timeout = timeout consular.fallback_timeout = fallback_timeout From 69affca8660faf02e43ea40ae81d72ad54207a61 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Wed, 9 Sep 2015 12:52:25 +0300 Subject: [PATCH 008/111] slightly better error handling --- consular/main.py | 8 +++++--- consular/tests/test_main.py | 9 ++++++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/consular/main.py b/consular/main.py index 0304dc8..5cb4bcb 100644 --- a/consular/main.py +++ b/consular/main.py @@ -105,8 +105,8 @@ def log_http_response(self, response, method, path, data): return response def log_http_error(self, failure, url): - log.msg('Error performing request to %s' % (url,)) - failure.raiseException() + log.err(failure, 'Error performing request to %s' % (url,)) + return failure def marathon_request(self, method, path, data=None): return self._request( @@ -126,9 +126,11 @@ def _request(self, method, url, data, timeout=None): data=(json.dumps(data) if data is not None else None), pool=self.pool, timeout=timeout or self.timeout) + if self.debug: d.addCallback(self.log_http_response, method, url, data) - d.addErrback(self.log_http_error, url) + + d.addErrback(self.log_http_error, url) return d @app.route('/') diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 70e81b1..441a51a 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -30,6 +30,10 @@ def json(self): return d +class DummyConsularException(Exception): + pass + + class ConsularTest(TestCase): timeout = 1 @@ -407,7 +411,10 @@ def test_fallback_to_main_consul(self): self.assertEqual( request['url'], 'http://foo:8500/v1/agent/service/register') - request['deferred'].errback(Exception('Something terrible')) + request['deferred'].errback( + DummyConsularException('Something terrible')) + [exc] = self.flushLoggedErrors(DummyConsularException) + self.assertEqual(str(exc.value), 'Something terrible') fallback_request = yield self.requests.get() self.assertEqual( From 851cb2bfcbb71628065c21a2134c1da60affbf71 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Wed, 9 Sep 2015 13:43:30 +0300 Subject: [PATCH 009/111] install latest pre crypto library for pypy<2.6 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 2c4f319..3b7adc0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ cache: directories: - $HOME/.cache/pip install: + - pip install "cryptography<=1.0" - pip install twine - pip install coveralls - pip install --upgrade pip From 2f3b6181905f317c269c3810636a2ee77aef2d38 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Wed, 9 Sep 2015 14:11:53 +0300 Subject: [PATCH 010/111] add comment about why the crypto pinning --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3b7adc0..544cb27 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ cache: directories: - $HOME/.cache/pip install: - - pip install "cryptography<=1.0" + - pip install "cryptography<=1.0" # NOTE: because pypy<2.5 on Travis - pip install twine - pip install coveralls - pip install --upgrade pip From dec91852f86693c913bd8da17c2b2a830f46972e Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Wed, 9 Sep 2015 14:31:13 +0300 Subject: [PATCH 011/111] documenting how to run tests --- docs/index.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 29f298e..ee44d30 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -41,6 +41,13 @@ Installing for local dev (ve)$ pip install -e . (ve)$ pip install -r requirements-dev.txt +Running tests +~~~~~~~~~~~~~ + +:: + + $ source ve/bin/activate + (ve)$ py.test consular .. _Marathon: http://mesosphere.github.io/marathon/ .. _Consul: http://consul.io/ From bf7fec334afe24713ae80e8bd27d74859955ba2c Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Wed, 9 Sep 2015 14:45:34 +0300 Subject: [PATCH 012/111] improve intro --- docs/index.rst | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index ee44d30..3e6b463 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,8 +1,17 @@ Consular ======== -Receive events from Marathon_, update Consul_ with the relevant information -about services & tasks. +Consular is a micro-service that relays information between Marathon_ and +Consul_. It registers itself for HTTP event callbacks with Marathon_ and makes +the appropriate API calls to register applications that Marathon_ runs as +services in Consul_. Registration of applications happens in the same way. + +Marathon_ is always considered the source of truth. + +If Marathon application definitions contain labels_ (application metadata) +they will be added to the Consul_ key/value store. This can be especially +useful when Consul_ is combined with `Consul Template`_ to automatically +generate configuration files for proxies such as HAProxy_ and Nginx_. .. image:: https://travis-ci.org/universalcore/consular.svg?branch=develop :target: https://travis-ci.org/universalcore/consular @@ -51,3 +60,7 @@ Running tests .. _Marathon: http://mesosphere.github.io/marathon/ .. _Consul: http://consul.io/ +.. _labels: https://mesosphere.github.io/marathon/docs/rest-api.html#labels-object-of-string-values +.. _HAProxy: http://www.haproxy.org/ +.. _Nginx: http://nginx.org/ +.. _`Consul Template`: https://github.com/hashicorp/consul-template From f6de3f56bbaf373e7631bce3ad628feb836bf046 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Wed, 9 Sep 2015 15:18:27 +0300 Subject: [PATCH 013/111] adding sequence diagram --- docs/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 3e6b463..ce84b72 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,6 +13,10 @@ they will be added to the Consul_ key/value store. This can be especially useful when Consul_ is combined with `Consul Template`_ to automatically generate configuration files for proxies such as HAProxy_ and Nginx_. +.. image:: http://www.websequencediagrams.com/cgi-bin/cdraw?lz=dGl0bGUgQ29uc3VsYXIsAAMHICYgTWFyYXRob24KCgACCCAtPgAgCTogTm90aWZpY2F0aW9uIG9mXG5uZXcgYXBwbAANBwoATAgALQo6IFJlZ2lzdHIAJw5zZXJ2aWNlABwVQWRkIEEASwogbWV0YWRhdGFcbihsYWJlbHMpdG8AgTcHXG5LL1Ygc3RvcgBDCACBJwotVGVtcGxhdGUAgSUTY2hhbmdlcwCBKQcAIQkgLT4gTG9hZC1CYWxhbmNlcjogR2VuZXJhdGUgbmV3XG5sb2FkLWIAFgcgY29uZmlnACIjUmVsb2FkACkHdQCCBQY&s=napkin + :target: http://www.websequencediagrams.com/?lz=dGl0bGUgQ29uc3VsYXIsAAMHICYgTWFyYXRob24KCgACCCAtPgAgCTogTm90aWZpY2F0aW9uIG9mXG5uZXcgYXBwbAANBwoATAgALQo6IFJlZ2lzdHIAJw5zZXJ2aWNlABwVQWRkIEEASwogbWV0YWRhdGFcbihsYWJlbHMpdG8AgTcHXG5LL1Ygc3RvcgBDCACBJwotVGVtcGxhdGUAgSUTY2hhbmdlcwCBKQcAIQkgLT4gTG9hZC1CYWxhbmNlcjogR2VuZXJhdGUgbmV3XG5sb2FkLWIAFgcgY29uZmlnACIjUmVsb2FkACkHdQCCBQY&s=napkin + alt: consular sequence diagram + .. image:: https://travis-ci.org/universalcore/consular.svg?branch=develop :target: https://travis-ci.org/universalcore/consular :alt: Continuous Integration From ef05d691e64cfeac3536ea500a9fbc472423769c Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Wed, 9 Sep 2015 15:23:02 +0300 Subject: [PATCH 014/111] fixing alt typo --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index ce84b72..3c8f217 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,7 +15,7 @@ generate configuration files for proxies such as HAProxy_ and Nginx_. .. image:: http://www.websequencediagrams.com/cgi-bin/cdraw?lz=dGl0bGUgQ29uc3VsYXIsAAMHICYgTWFyYXRob24KCgACCCAtPgAgCTogTm90aWZpY2F0aW9uIG9mXG5uZXcgYXBwbAANBwoATAgALQo6IFJlZ2lzdHIAJw5zZXJ2aWNlABwVQWRkIEEASwogbWV0YWRhdGFcbihsYWJlbHMpdG8AgTcHXG5LL1Ygc3RvcgBDCACBJwotVGVtcGxhdGUAgSUTY2hhbmdlcwCBKQcAIQkgLT4gTG9hZC1CYWxhbmNlcjogR2VuZXJhdGUgbmV3XG5sb2FkLWIAFgcgY29uZmlnACIjUmVsb2FkACkHdQCCBQY&s=napkin :target: http://www.websequencediagrams.com/?lz=dGl0bGUgQ29uc3VsYXIsAAMHICYgTWFyYXRob24KCgACCCAtPgAgCTogTm90aWZpY2F0aW9uIG9mXG5uZXcgYXBwbAANBwoATAgALQo6IFJlZ2lzdHIAJw5zZXJ2aWNlABwVQWRkIEEASwogbWV0YWRhdGFcbihsYWJlbHMpdG8AgTcHXG5LL1Ygc3RvcgBDCACBJwotVGVtcGxhdGUAgSUTY2hhbmdlcwCBKQcAIQkgLT4gTG9hZC1CYWxhbmNlcjogR2VuZXJhdGUgbmV3XG5sb2FkLWIAFgcgY29uZmlnACIjUmVsb2FkACkHdQCCBQY&s=napkin - alt: consular sequence diagram + :alt: consular sequence diagram .. image:: https://travis-ci.org/universalcore/consular.svg?branch=develop :target: https://travis-ci.org/universalcore/consular From ded53dfa2e5c7706439f2e104188ecf60f1c58ef Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Wed, 9 Sep 2015 15:25:24 +0300 Subject: [PATCH 015/111] fix typos --- docs/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 3c8f217..5efea57 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,8 +13,8 @@ they will be added to the Consul_ key/value store. This can be especially useful when Consul_ is combined with `Consul Template`_ to automatically generate configuration files for proxies such as HAProxy_ and Nginx_. -.. image:: http://www.websequencediagrams.com/cgi-bin/cdraw?lz=dGl0bGUgQ29uc3VsYXIsAAMHICYgTWFyYXRob24KCgACCCAtPgAgCTogTm90aWZpY2F0aW9uIG9mXG5uZXcgYXBwbAANBwoATAgALQo6IFJlZ2lzdHIAJw5zZXJ2aWNlABwVQWRkIEEASwogbWV0YWRhdGFcbihsYWJlbHMpdG8AgTcHXG5LL1Ygc3RvcgBDCACBJwotVGVtcGxhdGUAgSUTY2hhbmdlcwCBKQcAIQkgLT4gTG9hZC1CYWxhbmNlcjogR2VuZXJhdGUgbmV3XG5sb2FkLWIAFgcgY29uZmlnACIjUmVsb2FkACkHdQCCBQY&s=napkin - :target: http://www.websequencediagrams.com/?lz=dGl0bGUgQ29uc3VsYXIsAAMHICYgTWFyYXRob24KCgACCCAtPgAgCTogTm90aWZpY2F0aW9uIG9mXG5uZXcgYXBwbAANBwoATAgALQo6IFJlZ2lzdHIAJw5zZXJ2aWNlABwVQWRkIEEASwogbWV0YWRhdGFcbihsYWJlbHMpdG8AgTcHXG5LL1Ygc3RvcgBDCACBJwotVGVtcGxhdGUAgSUTY2hhbmdlcwCBKQcAIQkgLT4gTG9hZC1CYWxhbmNlcjogR2VuZXJhdGUgbmV3XG5sb2FkLWIAFgcgY29uZmlnACIjUmVsb2FkACkHdQCCBQY&s=napkin +.. image:: http://www.websequencediagrams.com/cgi-bin/cdraw?lz=dGl0bGUgQ29uc3VsYXIsAAMHICYgTWFyYXRob24KCgACCCAtPgAgCTogTm90aWZpY2F0aW9uIG9mXG5uZXcgYXBwbAANBwoATAgALQo6IFJlZ2lzdHIAJw5zZXJ2aWNlABwVQWRkIEEASwogbWV0YWRhdGFcbihsYWJlbHMpIHRvAIE4B1xuSy9WIHN0b3IARAgAgSgKLVRlbXBsYXRlAIEmE2NoYW5nZXMAgSoHACEJIC0-IExvYWQtQmFsYW5jZXI6IEdlbmVyYXRlIG5ld1xubG9hZC1iABYHIGNvbmZpZwAiI1JlbG9hZAApB3UAggYG&s=napkin + :target: http://www.websequencediagrams.com/?lz=dGl0bGUgQ29uc3VsYXIsAAMHICYgTWFyYXRob24KCgACCCAtPgAgCTogTm90aWZpY2F0aW9uIG9mXG5uZXcgYXBwbAANBwoATAgALQo6IFJlZ2lzdHIAJw5zZXJ2aWNlABwVQWRkIEEASwogbWV0YWRhdGFcbihsYWJlbHMpIHRvAIE4B1xuSy9WIHN0b3IARAgAgSgKLVRlbXBsYXRlAIEmE2NoYW5nZXMAgSoHACEJIC0-IExvYWQtQmFsYW5jZXI6IEdlbmVyYXRlIG5ld1xubG9hZC1iABYHIGNvbmZpZwAiI1JlbG9hZAApB3UAggYG&s=napkin :alt: consular sequence diagram .. image:: https://travis-ci.org/universalcore/consular.svg?branch=develop From 01ec40630deba41b6881ebfe8be83666feb4c75a Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Wed, 9 Sep 2015 15:38:44 +0300 Subject: [PATCH 016/111] typo --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 5efea57..6f2eaf3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,7 +4,7 @@ Consular Consular is a micro-service that relays information between Marathon_ and Consul_. It registers itself for HTTP event callbacks with Marathon_ and makes the appropriate API calls to register applications that Marathon_ runs as -services in Consul_. Registration of applications happens in the same way. +services in Consul_. De-registrations of applications happens in the same way. Marathon_ is always considered the source of truth. From 8949b7c11febf5bd4f476e998d6cd6dcf58055b9 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Wed, 9 Sep 2015 23:19:01 +0300 Subject: [PATCH 017/111] more docs --- consular/main.py | 102 ++++++++++++++++++++++++++++++++++++++++++++--- docs/conf.py | 7 +++- docs/index.rst | 7 ++++ 3 files changed, 108 insertions(+), 8 deletions(-) diff --git a/consular/main.py b/consular/main.py index f59b44e..634020a 100644 --- a/consular/main.py +++ b/consular/main.py @@ -32,6 +32,19 @@ def log(self, request): class Consular(object): + """ + :param str consul_endpoint: + The HTTP endpoint for Consul (often http://example.org:8500). + :param str marathon_endpoint: + The HTTP endpoint for Marathon (often http://example.org:8080). + :param bool enable_fallback: + Fallback to the main Consul endpoint for registrations if unable + to reach Consul running on the machine running a specific Marathon + task. + :param str registration_id: + A unique parameter for this Consul server. It is used for house-keeping + purposes such as purging tasks that are no longer running in Marathon. + """ app = Klein() debug = False @@ -52,6 +65,14 @@ def __init__(self, consul_endpoint, marathon_endpoint, enable_fallback, } def run(self, host, port): + """ + Starts the HTTP server. + + :param str host: + The host to bind to (example is ``localhost``) + :param int port: + The port to listen on (example is ``7000``) + """ site = ConsularSite(self.app.resource()) site.debug = self.debug self.clock.listenTCP(port, site, interface=host) @@ -64,12 +85,11 @@ def get_marathon_event_callbacks(self): return d def get_marathon_event_callbacks_from_json(self, json): - """ - Marathon may return a bad response when we get the existing event - callbacks. A common cause for this is that Marathon is not properly - configured. Raise an exception with information from Marathon if this - is the case, else return the callback URLs from the JSON response. - """ + # NOTE: + # Marathon may return a bad response when we get the existing event + # callbacks. A common cause for this is that Marathon is not properly + # configured. Raise an exception with information from Marathon if this + # is the case, else return the callback URLs from the JSON response. if 'callbackUrls' not in json: raise RuntimeError('Unable to get existing event callbacks from ' + 'Marathon: %r' % (str(json),)) @@ -87,6 +107,18 @@ def create_marathon_event_callback(self, url): @inlineCallbacks def register_marathon_event_callback(self, events_url): + """ + Register Consular with Marathon to receive HTTP event callbacks. + To use this ensure that `Marathon is configured`_ to send HTTP event + callbacks for state changes in tasks. + + :param str events_url: + The HTTP endpoint to register with Marathon for event callbacks. + + .. _`Marathon is configured`: + https://mesosphere.github.io/marathon/docs/event-bus.html + #configuration + """ existing_callbacks = yield self.get_marathon_event_callbacks() already_registered = any( [events_url == url for url in existing_callbacks]) @@ -142,6 +174,9 @@ def index(self, request): @app.route('/events') def events(self, request): + """ + Listens to incoming events from Marathon on ``/events``. + """ request.setHeader('Content-Type', 'application/json') event = json.load(request.content) handler = self.event_dispatch.get( @@ -149,6 +184,24 @@ def events(self, request): return handler(request, event) def handle_status_update_event(self, request, event): + """ + Handles status updates from Marathon. + + The various task stages are handled as follows: + + TASK_STAGING: ignored + TASK_STARTING: ignored + TASK_RUNNING: task data updated on Consul + TASK_FINISHED: task data removed from Consul + TASK_FAILED: task data removed from Consul + TASK_KILLED: task data removed from Consul + TASK_LOST: task data removed from Consul + + :param klein.app.KleinRequest request: + The Klein HTTP request + :param dict event: + The Marathon event + """ dispatch = { 'TASK_STAGING': self.noop, 'TASK_STARTING': self.noop, @@ -213,6 +266,21 @@ def _create_service_registration(self, app_id, service_id, address, port): def register_service(self, agent_endpoint, app_id, service_id, address, port): + """ + Register a task in Marathon as a service in Consul + + :param str agent_endpoint: + The HTTP endpoint of where Consul on the Mesos worker machine + can be accessed. + :param str app_id: + Marathon's App-id for the task. + :param str service_id: + The service-id to register it as in Consul. + :param str address: + The host address of the machine the task is running on. + :param int port: + The port number the task can be accessed on on the host machine. + """ log.msg('Registering %s at %s with %s at %s:%s.' % ( app_id, agent_endpoint, service_id, address, port)) registration = self._create_service_registration(app_id, service_id, @@ -235,6 +303,17 @@ def register_service_fallback(self, failure, registration): registration) def deregister_service(self, agent_endpoint, app_id, service_id): + """ + Deregister a service from Consul + + :param str agent_endpoint: + The HTTP endpoint of where Consul on the Mesos worker machine + can be accessed. + :param str app_id: + Marathon's App-id for the task. + :param str service_id: + The service-id to register it as in Consul. + """ log.msg('Deregistering %s at %s with %s' % ( app_id, agent_endpoint, service_id,)) return self.consul_request( @@ -242,6 +321,17 @@ def deregister_service(self, agent_endpoint, app_id, service_id): agent_endpoint, service_id,)) def sync_apps(self, purge=False): + """ + Ensure all the apps in Marathon are registered as services + in Consul. + + Set ``purge`` to ``True`` if you automatically want services in Consul + that aren't registered in Marathon to be purged. Consular only purges + services that have been registered with the same ``registration-id``. + + :param bool purge: + To purge or not to purge. + """ d = self.marathon_request('GET', '/v2/apps') d.addCallback(lambda response: response.json()) d.addCallback( diff --git a/docs/conf.py b/docs/conf.py index 4b22d4f..1a02c06 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -19,7 +19,7 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, os.path.abspath('..')) # -- General configuration ------------------------------------------------ @@ -289,4 +289,7 @@ # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/': None} +intersphinx_mapping = { + 'https://docs.python.org/': None, + 'klein': ('https://klein.readthedocs.org/en/latest/', None), +} diff --git a/docs/index.rst b/docs/index.rst index 6f2eaf3..9dfa853 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -68,3 +68,10 @@ Running tests .. _HAProxy: http://www.haproxy.org/ .. _Nginx: http://nginx.org/ .. _`Consul Template`: https://github.com/hashicorp/consul-template + + +Consular Class Documentation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: consular.main + :members: From 386292d3271613a9103850f677c3d077f73eeb7d Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Wed, 9 Sep 2015 23:24:38 +0300 Subject: [PATCH 018/111] fix event docstring --- consular/main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/consular/main.py b/consular/main.py index 634020a..58435e4 100644 --- a/consular/main.py +++ b/consular/main.py @@ -176,6 +176,9 @@ def index(self, request): def events(self, request): """ Listens to incoming events from Marathon on ``/events``. + + :param klein.app.KleinRequest request: + The Klein HTTP request """ request.setHeader('Content-Type', 'application/json') event = json.load(request.content) From 02a16f5bcba85bbf405012875d10b5384372da20 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Thu, 10 Sep 2015 09:51:35 +0200 Subject: [PATCH 019/111] Bump version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 6d7de6e..9084fa2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.2 +1.1.0 From 42c1b299236c34807b6f2d3ceafd342afcc62fca Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Thu, 10 Sep 2015 16:07:53 +0200 Subject: [PATCH 020/111] Support apps in groups * Replace '/' separators with '-' so a Marathon app at '/vumi-web/goui' results in a Consul service called 'vumi-web-goui'. * Tag apps with the Marathon app ID/path so that they can be identified in Marathon again. --- consular/main.py | 58 ++++++++++++++++++++++++++++++------- consular/tests/test_main.py | 17 ++++++++--- 2 files changed, 60 insertions(+), 15 deletions(-) diff --git a/consular/main.py b/consular/main.py index 58435e4..870ae63 100644 --- a/consular/main.py +++ b/consular/main.py @@ -14,8 +14,12 @@ from klein import Klein -def get_appid(app_id_string): - return app_id_string.rsplit('/', 1)[1] +def get_app_name(app_id): + """ + Get the app name from the marathon app ID. Separators in the ID ('/') are + replaced with '-'s while leading and trailing separators are removed. + """ + return app_id.strip('/').replace('/', '-') def get_agent_endpoint(host): @@ -233,7 +237,7 @@ def update_task_running(self, request, event): def update_task_killed(self, request, event): d = self.deregister_service( get_agent_endpoint(event['host']), - get_appid(event['appId']), + get_app_name(event['appId']), event['taskId']) d.addCallback(lambda _: json.dumps({'status': 'ok'})) return d @@ -254,16 +258,26 @@ def _registration_tag(self): """ return 'consular-reg-id:%s' % (self.registration_id,) + def _application_tag(self, app_id): + """ + Get the Consul service tag used to mark the Marathon path for a given + service. + """ + return 'consular-app-id:%s' % (app_id,) + def _create_service_registration(self, app_id, service_id, address, port): """ Create the request body for registering a service with Consul. """ registration = { - 'Name': app_id, + 'Name': get_app_name(app_id), 'ID': service_id, 'Address': address, 'Port': port, - 'Tags': [self._registration_tag()] + 'Tags': [ + self._registration_tag(), + self._application_tag(app_id), + ] } return registration @@ -364,7 +378,7 @@ def sync_app_labels(self, app): self.consul_request( 'PUT', '%s/v1/kv/consular/%s/%s' % ( self.consul_endpoint, - quote(get_appid(app['id'])), quote(key)), value) + quote(get_app_name(app['id'])), quote(key)), value) for key, value in labels.items() ]) @@ -377,8 +391,7 @@ def sync_app_tasks(self, app): def sync_app_task(self, app, task): return self.register_service( - get_agent_endpoint(task['host']), - get_appid(app['id']), task['id'], + get_agent_endpoint(task['host']), app['id'], task['id'], task['host'], task['ports'][0]) def purge_dead_services(self): @@ -401,9 +414,14 @@ def purge_dead_agent_services(self, agent_endpoint): services = {} for service_id, service in data.items(): # Check the service for a tag that matches our registration ID - if self._is_registration_in_tags(service['Tags']): - services.setdefault(service['Service'], set([])).add( - service_id) + tags = service['Tags'] + if self._is_registration_in_tags(tags): + # Get the app ID from the tags or default to the service name + app_id = self._get_app_id_from_tags(tags) + if not app_id: + app_id = service['Service'] + + services.setdefault(app_id, set()).add(service_id) for app_id, task_ids in services.items(): yield self.purge_service_if_dead(agent_endpoint, app_id, task_ids) @@ -417,6 +435,24 @@ def _is_registration_in_tags(self, tags): return self._registration_tag() in tags + def _get_app_id_from_tags(self, tags): + """ + Get the Marathon app ID from the Consular tags, or None if an app ID is + not found. + """ + if not tags: + return None + + matches = [tag for tag in tags if tag.startswith('consular-app-id:')] + + if not matches: + return None + if len(matches) > 1: + raise RuntimeError('Multiple (%d) Consular app IDs found: %s' + % len(matches), matches,) + + return matches[0].lstrip('consular-app-id:') + @inlineCallbacks def purge_service_if_dead(self, agent_endpoint, app_id, consul_task_ids): response = yield self.marathon_request( diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 3ab46a3..49254b6 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -174,7 +174,10 @@ def test_TASK_RUNNING(self): 'ID': 'my-app_0-1396592784349', 'Address': 'slave-1234.acme.org', 'Port': 31372, - 'Tags': ['consular-reg-id:test'], + 'Tags': [ + 'consular-reg-id:test', + 'consular-app-id:/my-app', + ], })) request['deferred'].callback( FakeResponse(200, [], json.dumps({}))) @@ -285,7 +288,10 @@ def test_sync_app_task(self): 'ID': 'my-task-id', 'Address': '0.0.0.0', 'Port': 1234, - 'Tags': ['consular-reg-id:test'], + 'Tags': [ + 'consular-reg-id:test', + 'consular-app-id:/my-app', + ], })) self.assertEqual(consul_request['method'], 'PUT') consul_request['deferred'].callback( @@ -487,7 +493,7 @@ def test_purge_old_services(self): def test_fallback_to_main_consul(self): self.consular.enable_fallback = True self.consular.register_service( - 'http://foo:8500', 'app_id', 'service_id', 'foo', 1234) + 'http://foo:8500', '/app_id', 'service_id', 'foo', 1234) request = yield self.requests.get() self.assertEqual( request['url'], @@ -506,5 +512,8 @@ def test_fallback_to_main_consul(self): 'ID': 'service_id', 'Address': 'foo', 'Port': 1234, - 'Tags': ['consular-reg-id:test'], + 'Tags': [ + 'consular-reg-id:test', + 'consular-app-id:/app_id', + ], })) From b25e7913de4431e0bd40dc01147d00c7507b8a7e Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Thu, 10 Sep 2015 16:28:12 +0200 Subject: [PATCH 021/111] Add test for creating service for app in group --- consular/tests/test_main.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 49254b6..9e53a31 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -298,6 +298,34 @@ def test_sync_app_task(self): FakeResponse(200, [], json.dumps({}))) yield d + @inlineCallbacks + def test_sync_app_task_grouped(self): + """ + When syncing an app in a group with a task, Consul is updated with a + service entry for the task. + """ + app = {'id': '/my-group/my-app'} + task = {'id': 'my-task-id', 'host': '0.0.0.0', 'ports': [1234]} + d = self.consular.sync_app_task(app, task) + consul_request = yield self.requests.get() + self.assertEqual( + consul_request['url'], + 'http://0.0.0.0:8500/v1/agent/service/register') + self.assertEqual(consul_request['data'], json.dumps({ + 'Name': 'my-group-my-app', + 'ID': 'my-task-id', + 'Address': '0.0.0.0', + 'Port': 1234, + 'Tags': [ + 'consular-reg-id:test', + 'consular-app-id:/my-group/my-app', + ], + })) + self.assertEqual(consul_request['method'], 'PUT') + consul_request['deferred'].callback( + FakeResponse(200, [], json.dumps({}))) + yield d + @inlineCallbacks def test_sync_app_labels(self): app = { From c7f47ffe88cdb1d8cf4c4b2d1915d472c56e604f Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Thu, 10 Sep 2015 16:52:27 +0200 Subject: [PATCH 022/111] Tweak app-id tagging behaviour and improve tests * Only purge services with app-id tags * Expect there to be a leading '/' in app IDs * Update purging tests with app-id tags --- consular/main.py | 13 +++++-------- consular/tests/test_main.py | 33 +++++++++++++++++++++------------ 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/consular/main.py b/consular/main.py index 870ae63..473c6bc 100644 --- a/consular/main.py +++ b/consular/main.py @@ -17,9 +17,9 @@ def get_app_name(app_id): """ Get the app name from the marathon app ID. Separators in the ID ('/') are - replaced with '-'s while leading and trailing separators are removed. + replaced with '-'s while the leading separator is removed. """ - return app_id.strip('/').replace('/', '-') + return app_id.lstrip('/').replace('/', '-') def get_agent_endpoint(host): @@ -416,12 +416,9 @@ def purge_dead_agent_services(self, agent_endpoint): # Check the service for a tag that matches our registration ID tags = service['Tags'] if self._is_registration_in_tags(tags): - # Get the app ID from the tags or default to the service name app_id = self._get_app_id_from_tags(tags) - if not app_id: - app_id = service['Service'] - - services.setdefault(app_id, set()).add(service_id) + if app_id: + services.setdefault(app_id, set()).add(service_id) for app_id, task_ids in services.items(): yield self.purge_service_if_dead(agent_endpoint, app_id, task_ids) @@ -456,7 +453,7 @@ def _get_app_id_from_tags(self, tags): @inlineCallbacks def purge_service_if_dead(self, agent_endpoint, app_id, consul_task_ids): response = yield self.marathon_request( - 'GET', '/v2/apps/%s/tasks' % (app_id,)) + 'GET', '/v2/apps%s/tasks' % (app_id,)) data = yield response.json() tasks_to_be_purged = set(consul_task_ids) if 'tasks' in data: diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 9e53a31..84cffad 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -390,21 +390,27 @@ def test_purge_dead_services(self): self.assertEqual(agent_request['method'], 'GET') agent_request['deferred'].callback( FakeResponse(200, [], json.dumps({ - "testingapp.someid1": { - "ID": "testingapp.someid1", + "testinggroup-someid1": { + "ID": "taskid1", "Service": "testingapp", "Tags": None, "Address": "machine-1", "Port": 8102, - 'Tags': ['consular-reg-id:test'], + "Tags": [ + "consular-reg-id:test", + "consular-app-id:/testinggroup/someid1", + ], }, - "testingapp.someid2": { - "ID": "testingapp.someid2", + "testinggroup-someid1": { + "ID": "taskid2", "Service": "testingapp", "Tags": None, "Address": "machine-2", "Port": 8103, - 'Tags': ['consular-reg-id:test'], + "Tags": [ + "consular-reg-id:test", + "consular-app-id:/testinggroup/someid1", + ], } })) ) @@ -413,13 +419,14 @@ def test_purge_dead_services(self): # 1 less than Consul thinks exists. testingapp_request = yield self.requests.get() self.assertEqual(testingapp_request['url'], - 'http://localhost:8080/v2/apps/testingapp/tasks') + 'http://localhost:8080/v2/apps/testinggroup/someid1/' + 'tasks') self.assertEqual(testingapp_request['method'], 'GET') testingapp_request['deferred'].callback( FakeResponse(200, [], json.dumps({ "tasks": [{ - "appId": "/testingapp", - "id": "testingapp.someid2", + "appId": "/testinggroup/someid1", + "id": "taskid2", "host": "machine-2", "ports": [8103], "startedAt": "2015-07-14T14:54:31.934Z", @@ -435,7 +442,7 @@ def test_purge_dead_services(self): self.assertEqual( deregister_request['url'], ('http://1.2.3.4:8500/v1/agent/service/deregister/' - 'testingapp.someid1')) + 'testinggroup-someid1')) self.assertEqual(deregister_request['method'], 'PUT') deregister_request['deferred'].callback( FakeResponse(200, [], json.dumps({}))) @@ -472,7 +479,8 @@ def test_purge_old_services(self): "ID": "testingapp.someid1", "Service": "testingapp", "Tags": [ - "consular-reg-id:test" + "consular-reg-id:test", + "consular-app-id:/testingapp", ], "Address": "machine-1", "Port": 8102 @@ -481,7 +489,8 @@ def test_purge_old_services(self): "ID": "testingapp.someid2", "Service": "testingapp", "Tags": [ - "consular-reg-id:blah" + "consular-reg-id:blah", + "consular-app-id:/testingapp", ], "Address": "machine-2", "Port": 8103 From 4f580c4f000aa933876bf359da37fa454256f1df Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Thu, 10 Sep 2015 17:33:56 +0200 Subject: [PATCH 023/111] Rework and refactor tagging in Consul * Refactor tag creation methods * Separate keys and values with '=' instead of ':' * Add tests specifically for tag creation --- consular/main.py | 75 +++++++++++++++++-------------------- consular/tests/test_main.py | 68 +++++++++++++++++++++++++-------- 2 files changed, 86 insertions(+), 57 deletions(-) diff --git a/consular/main.py b/consular/main.py index 473c6bc..8a5442f 100644 --- a/consular/main.py +++ b/consular/main.py @@ -251,19 +251,39 @@ def handle_unknown_event(self, request, event): 'error': 'Event type %s not supported.' % (event_type,) }) - def _registration_tag(self): - """ - Get the Consul service tag used to mark a service as created by this - instance of Consular. - """ - return 'consular-reg-id:%s' % (self.registration_id,) + def reg_id_tag(self): + """ Get the registration ID tag for this instance of Consular. """ + return self._consular_tag('reg-id', self.registration_id) + + def app_id_tag(self, app_id): + """ Get the app ID tag for the given app ID. """ + return self._consular_tag('app-id', app_id) + + def _consular_tag(self, tag_name, value): + return self._consular_tag_key(tag_name) + value - def _application_tag(self, app_id): + def get_app_id_from_tags(self, tags): """ - Get the Consul service tag used to mark the Marathon path for a given - service. + Get the app ID from the app ID tag in the given tags, or None if the + tag could not be found. """ - return 'consular-app-id:%s' % (app_id,) + return self._find_consular_tag(tags, 'app-id') + + def _find_consular_tag(self, tags, tag_name): + pseudo_key = self._consular_tag_key(tag_name) + matches = [tag for tag in tags if tag.startswith(pseudo_key)] + + if not matches: + return None + if len(matches) > 1: + raise RuntimeError('Multiple (%d) Consular tags found for key ' + '"%s": %s' + % (len(matches), pseudo_key, matches,)) + + return matches[0].lstrip(pseudo_key) + + def _consular_tag_key(self, tag_name): + return 'consular-%s=' % (tag_name,) def _create_service_registration(self, app_id, service_id, address, port): """ @@ -275,8 +295,8 @@ def _create_service_registration(self, app_id, service_id, address, port): 'Address': address, 'Port': port, 'Tags': [ - self._registration_tag(), - self._application_tag(app_id), + self.reg_id_tag(), + self.app_id_tag(app_id), ] } return registration @@ -415,41 +435,14 @@ def purge_dead_agent_services(self, agent_endpoint): for service_id, service in data.items(): # Check the service for a tag that matches our registration ID tags = service['Tags'] - if self._is_registration_in_tags(tags): - app_id = self._get_app_id_from_tags(tags) + if tags and self.reg_id_tag() in tags: + app_id = self.get_app_id_from_tags(tags) if app_id: services.setdefault(app_id, set()).add(service_id) for app_id, task_ids in services.items(): yield self.purge_service_if_dead(agent_endpoint, app_id, task_ids) - def _is_registration_in_tags(self, tags): - """ - Check if the Consul service was tagged with our registration ID. - """ - if not tags: - return False - - return self._registration_tag() in tags - - def _get_app_id_from_tags(self, tags): - """ - Get the Marathon app ID from the Consular tags, or None if an app ID is - not found. - """ - if not tags: - return None - - matches = [tag for tag in tags if tag.startswith('consular-app-id:')] - - if not matches: - return None - if len(matches) > 1: - raise RuntimeError('Multiple (%d) Consular app IDs found: %s' - % len(matches), matches,) - - return matches[0].lstrip('consular-app-id:') - @inlineCallbacks def purge_service_if_dead(self, agent_endpoint, app_id, consul_task_ids): response = yield self.marathon_request( diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 84cffad..f4bcccc 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -85,6 +85,42 @@ def request(self, method, path, data=None): def tearDown(self): pass + def test_reg_id_tag(self): + self.assertEqual(self.consular.reg_id_tag(), 'consular-reg-id=test') + + def test_app_id_tag(self): + self.assertEqual(self.consular.app_id_tag('test'), + 'consular-app-id=test') + + def test_get_app_id_from_tags(self): + tags = [ + 'randomstuff', + 'consular-reg-id=test', + 'consular-app-id=/my-app', + ] + self.assertEqual(self.consular.get_app_id_from_tags(tags), '/my-app') + + def test_get_app_id_from_tags_not_found(self): + tags = [ + 'randomstuff', + 'consular-reg-id=test', + ] + self.assertEqual(self.consular.get_app_id_from_tags(tags), None) + + def test_get_app_id_from_tags_multiple(self): + tags = [ + 'randomstuff', + 'consular-reg-id=test', + 'consular-app-id=/my-app', + 'consular-app-id=/my-app2', + ] + exception = self.assertRaises(RuntimeError, + self.consular.get_app_id_from_tags, tags) + self.assertEqual(str(exception), + 'Multiple (2) Consular tags found for key ' + '"consular-app-id=": [\'consular-app-id=/my-app\', ' + '\'consular-app-id=/my-app2\']') + @inlineCallbacks def test_service(self): response = yield self.request('GET', '/') @@ -175,8 +211,8 @@ def test_TASK_RUNNING(self): 'Address': 'slave-1234.acme.org', 'Port': 31372, 'Tags': [ - 'consular-reg-id:test', - 'consular-app-id:/my-app', + 'consular-reg-id=test', + 'consular-app-id=/my-app', ], })) request['deferred'].callback( @@ -289,8 +325,8 @@ def test_sync_app_task(self): 'Address': '0.0.0.0', 'Port': 1234, 'Tags': [ - 'consular-reg-id:test', - 'consular-app-id:/my-app', + 'consular-reg-id=test', + 'consular-app-id=/my-app', ], })) self.assertEqual(consul_request['method'], 'PUT') @@ -317,8 +353,8 @@ def test_sync_app_task_grouped(self): 'Address': '0.0.0.0', 'Port': 1234, 'Tags': [ - 'consular-reg-id:test', - 'consular-app-id:/my-group/my-app', + 'consular-reg-id=test', + 'consular-app-id=/my-group/my-app', ], })) self.assertEqual(consul_request['method'], 'PUT') @@ -397,8 +433,8 @@ def test_purge_dead_services(self): "Address": "machine-1", "Port": 8102, "Tags": [ - "consular-reg-id:test", - "consular-app-id:/testinggroup/someid1", + "consular-reg-id=test", + "consular-app-id=/testinggroup/someid1", ], }, "testinggroup-someid1": { @@ -408,8 +444,8 @@ def test_purge_dead_services(self): "Address": "machine-2", "Port": 8103, "Tags": [ - "consular-reg-id:test", - "consular-app-id:/testinggroup/someid1", + "consular-reg-id=test", + "consular-app-id=/testinggroup/someid1", ], } })) @@ -479,8 +515,8 @@ def test_purge_old_services(self): "ID": "testingapp.someid1", "Service": "testingapp", "Tags": [ - "consular-reg-id:test", - "consular-app-id:/testingapp", + "consular-reg-id=test", + "consular-app-id=/testingapp", ], "Address": "machine-1", "Port": 8102 @@ -489,8 +525,8 @@ def test_purge_old_services(self): "ID": "testingapp.someid2", "Service": "testingapp", "Tags": [ - "consular-reg-id:blah", - "consular-app-id:/testingapp", + "consular-reg-id=blah", + "consular-app-id=/testingapp", ], "Address": "machine-2", "Port": 8103 @@ -550,7 +586,7 @@ def test_fallback_to_main_consul(self): 'Address': 'foo', 'Port': 1234, 'Tags': [ - 'consular-reg-id:test', - 'consular-app-id:/app_id', + 'consular-reg-id=test', + 'consular-app-id=/app_id', ], })) From e47aba1a037b41ed43b0f4786d4cda3e5c294b6a Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Thu, 10 Sep 2015 17:48:20 +0200 Subject: [PATCH 024/111] Improve logging when checking tags --- consular/main.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/consular/main.py b/consular/main.py index 8a5442f..0327e42 100644 --- a/consular/main.py +++ b/consular/main.py @@ -439,6 +439,13 @@ def purge_dead_agent_services(self, agent_endpoint): app_id = self.get_app_id_from_tags(tags) if app_id: services.setdefault(app_id, set()).add(service_id) + else: + log.msg('Service "%s" does not have an app ID in its ' + 'tags, it cannot be purged.' + % (service['Service'],)) + elif self.debug: + log.msg('Service "%s" is not tagged with our registration ID, ' + 'not touching it.' % (service['Service'],)) for app_id, task_ids in services.items(): yield self.purge_service_if_dead(agent_endpoint, app_id, task_ids) From b8e12b5fd105b87b929e56e536b51d108810b662 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Thu, 10 Sep 2015 17:51:52 +0200 Subject: [PATCH 025/111] Add docstrings to new tests --- consular/tests/test_main.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index f4bcccc..4778b34 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -86,13 +86,16 @@ def tearDown(self): pass def test_reg_id_tag(self): + """ Consular's registration ID tag is properly formed. """ self.assertEqual(self.consular.reg_id_tag(), 'consular-reg-id=test') def test_app_id_tag(self): + """ Consular's application ID tag is properly formed. """ self.assertEqual(self.consular.app_id_tag('test'), 'consular-app-id=test') def test_get_app_id_from_tags(self): + """ The app ID is successfully parsed from the Consul tags. """ tags = [ 'randomstuff', 'consular-reg-id=test', @@ -101,6 +104,9 @@ def test_get_app_id_from_tags(self): self.assertEqual(self.consular.get_app_id_from_tags(tags), '/my-app') def test_get_app_id_from_tags_not_found(self): + """ + None is returned when the app ID cannot be found in the Consul tags. + """ tags = [ 'randomstuff', 'consular-reg-id=test', @@ -108,6 +114,10 @@ def test_get_app_id_from_tags_not_found(self): self.assertEqual(self.consular.get_app_id_from_tags(tags), None) def test_get_app_id_from_tags_multiple(self): + """ + An exception is raised when multiple app IDs are found in the Consul + tags. + """ tags = [ 'randomstuff', 'consular-reg-id=test', From ad287e511f5815404dc69f1438597aac55b68230 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Thu, 10 Sep 2015 18:05:32 +0200 Subject: [PATCH 026/111] Thou shalt not decrease test coverage --- consular/tests/test_main.py | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 4778b34..da88988 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -572,6 +572,45 @@ def test_purge_old_services(self): FakeResponse(200, [], json.dumps({}))) yield d + @inlineCallbacks + def test_purge_old_service_no_app_id(self): + """ + Services previously registered with Consul by Consular but without an + app ID tagged (for some reason) should not be purged. + """ + d = self.consular.purge_dead_services() + consul_request = yield self.requests.get() + self.assertEqual( + consul_request['url'], + 'http://localhost:8500/v1/catalog/nodes') + consul_request['deferred'].callback( + FakeResponse(200, [], json.dumps([{ + 'Node': 'consul-node', + 'Address': '1.2.3.4', + }])) + ) + agent_request = yield self.requests.get() + self.assertEqual( + agent_request['url'], + 'http://1.2.3.4:8500/v1/agent/services') + self.assertEqual(agent_request['method'], 'GET') + agent_request['deferred'].callback( + FakeResponse(200, [], json.dumps({ + "testingapp.someid1": { + "ID": "testingapp.someid1", + "Service": "testingapp", + "Tags": [ + "consular-reg-id=test", + ], + "Address": "machine-1", + "Port": 8102 + } + })) + ) + + # Expecting no action to be taken as there is no app ID. + yield d + @inlineCallbacks def test_fallback_to_main_consul(self): self.consular.enable_fallback = True From 1d472f63cc7375b84d6e98767b60d44a7f68615c Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Mon, 14 Sep 2015 12:25:17 +0200 Subject: [PATCH 027/111] Add check for Marathon/Consul app/service namespace clash * Throw an error if there is a clash --- consular/main.py | 34 ++++++++++++++++++++++++++++++++-- consular/tests/test_main.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/consular/main.py b/consular/main.py index 0327e42..7859f5a 100644 --- a/consular/main.py +++ b/consular/main.py @@ -371,13 +371,43 @@ def sync_apps(self, purge=False): """ d = self.marathon_request('GET', '/v2/apps') d.addCallback(lambda response: response.json()) + d.addCallback(lambda response_json: response_json['apps']) + d.addCallback(self.check_apps_namespace_clash) d.addCallback( - lambda data: gatherResults( - [self.sync_app(app) for app in data['apps']])) + lambda apps: gatherResults([self.sync_app(app) for app in apps])) if purge: d.addCallback(lambda _: self.purge_dead_services()) return d + def check_apps_namespace_clash(self, apps): + """ + Checks if app names in Marathon will cause a namespace clash in Consul. + Throws an exception if there is a collision, else returns the apps. + + :param: apps: + The JSON list of apps from Marathon's API. + """ + # Collect the app name to app id(s) mapping. + name_ids = {} + for app in apps: + app_id = app['id'] + app_name = get_app_name(app_id) + name_ids.setdefault(app_name, []).append(app_id) + + # Check if any app names map to more than one app id. + collisions = {name: ids + for name, ids in name_ids.items() if len(ids) > 1} + + if collisions: + collisions_string = '\n'.join(sorted( + ['%s => %s' % (name, ', '.join(ids),) + for name, ids in collisions.items()])) + raise RuntimeError( + 'The following Consul service name will resolve to multiple ' + 'Marathon app names: \n%s' % (collisions_string,)) + + return apps + def get_app(self, app_id): d = self.marathon_request('GET', '/v2/apps%s' % (app_id,)) d.addCallback(lambda response: response.json()) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index da88988..1cabf2e 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -414,6 +414,39 @@ def test_sync_apps(self): FakeResponse(200, [], json.dumps({'apps': []}))) yield d + def test_check_apps_namespace_clash_no_clash(self): + """ + When checking for app namespace clashes and there are no clashes, the + list of apps is returned. + """ + apps = [ + {'id': '/my-group/my-app'}, + {'id': '/my-app'}, + {'id': '/my-group/my-app2'}, + ] + apps_returned = self.consular.check_apps_namespace_clash(apps) + self.assertEqual(apps, apps_returned) + + def test_check_apps_namespace_clash_clashing(self): + """ + When checking for app namespace clashes and there are clashes, an + error is raised with an error message describing the clashes. + """ + apps = [ + {'id': '/my-group/my-subgroup/my-app'}, + {'id': '/my-group/my-subgroup-my-app'}, + {'id': '/my-group-my-subgroup-my-app'}, + {'id': '/my-app'}, + ] + exception = self.assertRaises( + RuntimeError, self.consular.check_apps_namespace_clash, apps) + + self.assertEqual('The following Consul service name will resolve to ' + 'multiple Marathon app names: \nmy-group-my-subgroup-' + 'my-app => /my-group/my-subgroup/my-app, /my-group/my' + '-subgroup-my-app, /my-group-my-subgroup-my-app', + str(exception)) + @inlineCallbacks def test_purge_dead_services(self): d = self.consular.purge_dead_services() From 299890884813d945546009cf6bac406314122db4 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Mon, 14 Sep 2015 12:35:14 +0200 Subject: [PATCH 028/111] Plurality in error message --- consular/main.py | 4 ++-- consular/tests/test_main.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/consular/main.py b/consular/main.py index 7859f5a..4acc551 100644 --- a/consular/main.py +++ b/consular/main.py @@ -403,8 +403,8 @@ def check_apps_namespace_clash(self, apps): ['%s => %s' % (name, ', '.join(ids),) for name, ids in collisions.items()])) raise RuntimeError( - 'The following Consul service name will resolve to multiple ' - 'Marathon app names: \n%s' % (collisions_string,)) + 'The following Consul service name(s) will resolve to ' + 'multiple Marathon app names: \n%s' % (collisions_string,)) return apps diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 1cabf2e..ae9b816 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -441,10 +441,10 @@ def test_check_apps_namespace_clash_clashing(self): exception = self.assertRaises( RuntimeError, self.consular.check_apps_namespace_clash, apps) - self.assertEqual('The following Consul service name will resolve to ' - 'multiple Marathon app names: \nmy-group-my-subgroup-' - 'my-app => /my-group/my-subgroup/my-app, /my-group/my' - '-subgroup-my-app, /my-group-my-subgroup-my-app', + self.assertEqual('The following Consul service name(s) will resolve ' + 'to multiple Marathon app names: \nmy-group-my-subgro' + 'up-my-app => /my-group/my-subgroup/my-app, /my-group' + '/my-subgroup-my-app, /my-group-my-subgroup-my-app', str(exception)) @inlineCallbacks From e05317723cb93b7551a041a6833da8c9a8822f90 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Mon, 14 Sep 2015 19:27:49 +0200 Subject: [PATCH 029/111] Clean up app labels during app sync * Delete old labels from Consul --- consular/main.py | 96 ++++++++++++++++++++++++++++++++++--- consular/tests/test_main.py | 80 +++++++++++++++++++++++++++++++ 2 files changed, 169 insertions(+), 7 deletions(-) diff --git a/consular/main.py b/consular/main.py index 0327e42..ae93aa2 100644 --- a/consular/main.py +++ b/consular/main.py @@ -388,20 +388,102 @@ def sync_app(self, app): return gatherResults([ self.sync_app_labels(app), self.sync_app_tasks(app), - ]) + ]) def sync_app_labels(self, app): - labels = app.get('labels', {}) + """ + Sync the app labels for the given app by pushing its labels to the + Consul k/v store and cleaning any labels there that are no longer + present. + + :param: app: + The app JSON as return by the Marathon HTTP API. + """ # NOTE: KV requests can go straight to the consul registry # we're already connected to, they're not local to the agents. + app_name = get_app_name(app['id']) + labels = app.get('labels', {}) return gatherResults([ - self.consul_request( - 'PUT', '%s/v1/kv/consular/%s/%s' % ( - self.consul_endpoint, - quote(get_app_name(app['id'])), quote(key)), value) - for key, value in labels.items() + self.put_consul_app_labels(app_name, labels), + self.clean_consul_app_labels(app_name, labels) ]) + def put_consul_app_labels(self, app_name, labels): + """ + Store the given set of labels under the given app name in the Consul + k/v store. + """ + return self.put_consul_kvs({'consular/%s/%s' % (app_name, key,): value + for key, value in labels.items()}) + + def put_consul_kvs(self, key_values): + """ Store the given key/value set in the Consul k/v store. """ + return gatherResults([self.put_consul_kv(key, value) + for key, value in key_values.items()]) + + def put_consul_kv(self, key, value): + """ Store the given value at the given key in the Consul k/v store. """ + return self.consul_request('PUT', '%s/v1/kv/%s' % ( + self.consul_endpoint, quote(key),), value) + + def clean_consul_app_labels(self, app_name, labels): + """ + Delete app labels stored in the Consul k/v store under the given app + name that aren't present in the given set of labels. + """ + # Get the existing labels from Consul + d = self.get_consul_app_keys(app_name) + + # Filter out the Marathon labels + d.addCallback(self._filter_marathon_labels, labels) + + # Delete the non-existant keys + d.addCallback(self.delete_consul_kv_keys) + + return d + + def get_consul_app_keys(self, app_name): + """ Get the Consul k/v keys for the app with the given name. """ + return self.get_consul_kv_keys('consular/%s' % (app_name,)) + + def get_consul_kv_keys(self, key_path): + """ Get the Consul k/v keys present at the given key path. """ + d = self.consul_request('GET', '%s/v1/kv/%s?keys' % ( + self.consul_endpoint, quote(key_path),)) + d.addCallback(lambda response: response.json()) + return d + + def delete_consul_kv_keys(self, keys): + """ Delete a sequence of Consul k/v keys. """ + return gatherResults([self.delete_consul_kv_key(key) for key in keys]) + + def delete_consul_kv_key(self, key): + """ Delete the Consul k/v entry associated with the given key. """ + return self.consul_request('DELETE', '%s/v1/kv/%s' % ( + self.consul_endpoint, quote(key),)) + + def _filter_marathon_labels(self, consul_keys, marathon_labels): + """ + Takes a list of Consul keys and removes those with keys not found in + the given dict of Marathon labels. + + :param: consul_keys: + The list of Consul keys as returned by the Consul API. + :param: marathon_labels: + The dict of Marathon labels as returned by the Marathon API. + """ + label_key_set = set(marathon_labels.keys()) + return [key for key in consul_keys + if (self._consul_key_to_marathon_label_key(key) + not in label_key_set)] + + def _consul_key_to_marathon_label_key(self, consul_key): + """ + Trims the 'consular//' from the front of the key path to get + the Marathon label key. + """ + return consul_key.split('/', 2)[-1] + def sync_app_tasks(self, app): d = self.marathon_request('GET', '/v2/apps%(id)s/tasks' % app) d.addCallback(lambda response: response.json()) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index da88988..0d8e9f0 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -196,6 +196,14 @@ def test_TASK_RUNNING(self): } }))) + # Check if any existing labels stored in Consul + consul_kv_request = yield self.requests.get() + self.assertEqual(consul_kv_request['method'], 'GET') + self.assertEqual(consul_kv_request['url'], + 'http://localhost:8500/v1/kv/consular/my-app?keys') + consul_kv_request['deferred'].callback( + FakeResponse(200, [], json.dumps([]))) + # Then we collect the tasks for the app marathon_tasks_request = yield self.requests.get() self.assertEqual(marathon_tasks_request['method'], 'GET') @@ -386,6 +394,63 @@ def test_sync_app_labels(self): self.assertEqual(consul_request['data'], '"bar"') consul_request['deferred'].callback( FakeResponse(200, [], json.dumps({}))) + + consul_request = yield self.requests.get() + self.assertEqual(consul_request['method'], 'GET') + self.assertEqual(consul_request['url'], + 'http://localhost:8500/v1/kv/consular/my-app?keys') + consul_request['deferred'].callback( + FakeResponse(200, [], json.dumps([]))) + + yield d + + @inlineCallbacks + def test_sync_app_labels_cleanup(self): + """ + When Consular syncs app labels, and labels are found in Consul which + aren't present in the Marathon app definition, those labels are deleted + from Consul. + """ + app = { + 'id': '/my-app', + 'labels': {'foo': 'bar'} + } + d = self.consular.sync_app_labels(app) + put_request = yield self.requests.get() + self.assertEqual(put_request['method'], 'PUT') + self.assertEqual(put_request['url'], + 'http://localhost:8500/v1/kv/consular/my-app/foo') + self.assertEqual(put_request['data'], '"bar"') + put_request['deferred'].callback( + FakeResponse(200, [], json.dumps({}))) + + get_request = yield self.requests.get() + self.assertEqual(get_request['method'], 'GET') + self.assertEqual(get_request['url'], + 'http://localhost:8500/v1/kv/consular/my-app?keys') + consul_labels = [ + 'consular/my-app/foo', + 'consular/my-app/oldfoo', + 'consular/my-app/misplaced/foo', + ] + get_request['deferred'].callback( + FakeResponse(200, [], json.dumps(consul_labels))) + + delete_request1 = yield self.requests.get() + self.assertEqual(delete_request1['method'], 'DELETE') + self.assertEqual(delete_request1['url'], + 'http://localhost:8500/v1/kv/consular/my-app/oldfoo') + delete_request1['deferred'].callback( + FakeResponse(200, [], json.dumps(True))) + + delete_request2 = yield self.requests.get() + self.assertEqual(delete_request2['method'], 'DELETE') + self.assertEqual( + delete_request2['url'], + 'http://localhost:8500/v1/kv/consular/my-app/misplaced/foo') + delete_request2['deferred'].callback( + FakeResponse(200, [], json.dumps(True))) + yield d @inlineCallbacks @@ -394,6 +459,21 @@ def test_sync_app(self): 'id': '/my-app', } d = self.consular.sync_app(app) + + # First Consular syncs app labels... + # There are no labels in this definition so Consular doesn't push any + # labels to Consul, it just tries to clean up any existing labels. + consul_request = yield self.requests.get() + self.assertEqual(consul_request['method'], 'GET') + self.assertEqual( + consul_request['url'], + 'http://localhost:8500/v1/kv/consular/my-app?keys') + consul_request['deferred'].callback( + FakeResponse(200, [], json.dumps([]))) + + # Next Consular syncs app tasks... + # It fetches a list of tasks for an app and gets an empty list so + # nothing is to be done. marathon_request = yield self.requests.get() self.assertEqual( marathon_request['url'], From 8590ddc1c6f414820b751ae058ad4984a7ae6dee Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 15 Sep 2015 17:00:08 +0200 Subject: [PATCH 030/111] Delete Consul k/v entries for non-existant apps * With `--purge`, clean out old k/v entries for apps during every sync. --- consular/main.py | 106 ++++++++++++++++++++++++++++-------- consular/tests/test_main.py | 42 ++++++++++++-- 2 files changed, 120 insertions(+), 28 deletions(-) diff --git a/consular/main.py b/consular/main.py index ae93aa2..bbf95ac 100644 --- a/consular/main.py +++ b/consular/main.py @@ -369,20 +369,25 @@ def sync_apps(self, purge=False): :param bool purge: To purge or not to purge. """ + d = self.get_marathon_apps() + return d.addCallback(self.sync_and_purge_apps, purge) + + def get_marathon_apps(self): + """ Get a list of running apps from the Marathon API. """ d = self.marathon_request('GET', '/v2/apps') d.addCallback(lambda response: response.json()) - d.addCallback( - lambda data: gatherResults( - [self.sync_app(app) for app in data['apps']])) + return d.addCallback(lambda data: data['apps']) + + def sync_and_purge_apps(self, apps, purge=False): + deferreds = [gatherResults([self.sync_app(app) for app in apps])] if purge: - d.addCallback(lambda _: self.purge_dead_services()) - return d + deferreds.append(self.purge_dead_apps(apps)) + return gatherResults(deferreds) def get_app(self, app_id): d = self.marathon_request('GET', '/v2/apps%s' % (app_id,)) - d.addCallback(lambda response: response.json()) - d.addCallback(lambda data: data['app']) - return d + d = d.addCallback(lambda response: response.json()) + return d.addCallback(lambda data: data['app']) def sync_app(self, app): return gatherResults([ @@ -390,6 +395,12 @@ def sync_app(self, app): self.sync_app_tasks(app), ]) + def purge_dead_apps(self, apps): + return gatherResults([ + self.purge_dead_services(), + self.purge_dead_app_labels(apps) + ]) + def sync_app_labels(self, app): """ Sync the app labels for the given app by pushing its labels to the @@ -438,29 +449,38 @@ def clean_consul_app_labels(self, app_name, labels): d.addCallback(self._filter_marathon_labels, labels) # Delete the non-existant keys - d.addCallback(self.delete_consul_kv_keys) - - return d + return d.addCallback(self.delete_consul_kv_keys) def get_consul_app_keys(self, app_name): """ Get the Consul k/v keys for the app with the given name. """ return self.get_consul_kv_keys('consular/%s' % (app_name,)) - def get_consul_kv_keys(self, key_path): - """ Get the Consul k/v keys present at the given key path. """ - d = self.consul_request('GET', '%s/v1/kv/%s?keys' % ( - self.consul_endpoint, quote(key_path),)) - d.addCallback(lambda response: response.json()) - return d + def get_consul_consular_keys(self): + """ + Get the next level of Consul k/v keys at 'consular/', i.e. will + return 'consular/my-app' but not 'consular/my-app/my-label'. + """ + return self.get_consul_kv_keys('consular/', separator='/') - def delete_consul_kv_keys(self, keys): + def get_consul_kv_keys(self, key_path, separator=None): + """ Get the Consul k/v keys present at the given key path. """ + params = {'keys': ''} + if separator: + params['separator'] = separator + d = self.consul_request('GET', '%s/v1/kv/%s?%s' % ( + self.consul_endpoint, quote(key_path), urlencode(params))) + return d.addCallback(lambda response: response.json()) + + def delete_consul_kv_keys(self, keys, recurse=False): """ Delete a sequence of Consul k/v keys. """ - return gatherResults([self.delete_consul_kv_key(key) for key in keys]) + return gatherResults([self.delete_consul_kv_key(key, recurse) + for key in keys]) - def delete_consul_kv_key(self, key): + def delete_consul_kv_key(self, key, recurse=False): """ Delete the Consul k/v entry associated with the given key. """ - return self.consul_request('DELETE', '%s/v1/kv/%s' % ( - self.consul_endpoint, quote(key),)) + return self.consul_request('DELETE', '%s/v1/kv/%s%s' % ( + self.consul_endpoint, quote(key), + '?recurse' if recurse else '',)) def _filter_marathon_labels(self, consul_keys, marathon_labels): """ @@ -487,15 +507,53 @@ def _consul_key_to_marathon_label_key(self, consul_key): def sync_app_tasks(self, app): d = self.marathon_request('GET', '/v2/apps%(id)s/tasks' % app) d.addCallback(lambda response: response.json()) - d.addCallback(lambda data: gatherResults( + return d.addCallback(lambda data: gatherResults( self.sync_app_task(app, task) for task in data['tasks'])) - return d def sync_app_task(self, app, task): return self.register_service( get_agent_endpoint(task['host']), app['id'], task['id'], task['host'], task['ports'][0]) + def purge_dead_app_labels(self, apps): + """ + Delete any keys stored in the Consul k/v store that belong to apps that + no longer exist. + + :param: apps: + The list of apps as returned by the Marathon API. + """ + # Get the existing keys + d = self.get_consul_consular_keys() + + # Filter the present apps out + d.addCallback(self._filter_marathon_apps, apps) + + # Delete the remaining keys + return d.addCallback(self.delete_consul_kv_keys, recurse=True) + + def _filter_marathon_apps(self, consul_keys, marathon_apps): + """ + Takes a list of Consul keys and removes those with keys not found in + the given list of Marathon apps. + + :param: consul_keys: + The list of Consul keys as returned by the Consul API. + :param: marathon_apps: + The list of apps as returned by the Marathon API. + """ + app_name_set = set([get_app_name(app['id']) for app in marathon_apps]) + return [key for key in consul_keys + if (self._consul_key_to_marathon_app_name(key) + not in app_name_set)] + + def _consul_key_to_marathon_app_name(self, consul_key): + """ + Trims the 'consular/' from the front of the key path to get the + Marathon app name. + """ + return consul_key.split('/', 1)[-1] + def purge_dead_services(self): d = self.consul_request( 'GET', '%s/v1/catalog/nodes' % (self.consul_endpoint,)) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 0d8e9f0..4b2534e 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -200,7 +200,7 @@ def test_TASK_RUNNING(self): consul_kv_request = yield self.requests.get() self.assertEqual(consul_kv_request['method'], 'GET') self.assertEqual(consul_kv_request['url'], - 'http://localhost:8500/v1/kv/consular/my-app?keys') + 'http://localhost:8500/v1/kv/consular/my-app?keys=') consul_kv_request['deferred'].callback( FakeResponse(200, [], json.dumps([]))) @@ -398,7 +398,7 @@ def test_sync_app_labels(self): consul_request = yield self.requests.get() self.assertEqual(consul_request['method'], 'GET') self.assertEqual(consul_request['url'], - 'http://localhost:8500/v1/kv/consular/my-app?keys') + 'http://localhost:8500/v1/kv/consular/my-app?keys=') consul_request['deferred'].callback( FakeResponse(200, [], json.dumps([]))) @@ -427,7 +427,7 @@ def test_sync_app_labels_cleanup(self): get_request = yield self.requests.get() self.assertEqual(get_request['method'], 'GET') self.assertEqual(get_request['url'], - 'http://localhost:8500/v1/kv/consular/my-app?keys') + 'http://localhost:8500/v1/kv/consular/my-app?keys=') consul_labels = [ 'consular/my-app/foo', 'consular/my-app/oldfoo', @@ -467,7 +467,7 @@ def test_sync_app(self): self.assertEqual(consul_request['method'], 'GET') self.assertEqual( consul_request['url'], - 'http://localhost:8500/v1/kv/consular/my-app?keys') + 'http://localhost:8500/v1/kv/consular/my-app?keys=') consul_request['deferred'].callback( FakeResponse(200, [], json.dumps([]))) @@ -691,6 +691,40 @@ def test_purge_old_service_no_app_id(self): # Expecting no action to be taken as there is no app ID. yield d + @inlineCallbacks + def test_purge_dead_app_labels(self): + """ + Services previously registered with Consul by Consular but that no + longer exist in Marathon should have their labels removed from the k/v + store. + """ + d = self.consular.purge_dead_app_labels([{ + 'id': 'my-app' + }]) + consul_request = yield self.requests.get() + self.assertEqual(consul_request['method'], 'GET') + self.assertEqual( + consul_request['url'], + 'http://localhost:8500/v1/kv/consular/?keys=&separator=%2F') + # Return one existing app and one non-existing app + consul_request['deferred'].callback( + FakeResponse(200, [], json.dumps([ + 'consular/my-app', + 'consular/my-app2', + ])) + ) + + # Consular should delete the app that doesn't exist + consul_request = yield self.requests.get() + self.assertEqual(consul_request['method'], 'DELETE') + self.assertEqual( + consul_request['url'], + 'http://localhost:8500/v1/kv/consular/my-app2?recurse') + consul_request['deferred'].callback( + FakeResponse(200, [], json.dumps({}))) + + yield d + @inlineCallbacks def test_fallback_to_main_consul(self): self.consular.enable_fallback = True From 88d4107dad8f6191a524f90fd0eed6965c71dda3 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Wed, 16 Sep 2015 10:56:52 +0200 Subject: [PATCH 031/111] Bump version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9084fa2..524cb55 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.0 +1.1.1 From 7f9e404e525acdf303106f36651c74b08e86f0ca Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Wed, 16 Sep 2015 15:32:07 +0200 Subject: [PATCH 032/111] Refactor Marathon client into own class --- consular/cli.py | 4 +- consular/clients.py | 104 ++++++++++++++++++++++++++++++ consular/main.py | 122 +++++++++++++++--------------------- consular/tests/test_main.py | 18 +++--- 4 files changed, 166 insertions(+), 82 deletions(-) create mode 100644 consular/clients.py diff --git a/consular/cli.py b/consular/cli.py index 08406ca..f516cbc 100644 --- a/consular/cli.py +++ b/consular/cli.py @@ -61,8 +61,8 @@ def main(scheme, host, port, log.startLogging(logfile) consular = Consular(consul, marathon, fallback, registration_id) - consular.debug = debug - consular.timeout = timeout + consular.set_debug(debug) + consular.set_timeout(timeout) consular.fallback_timeout = fallback_timeout events_url = "%s://%s:%s/events?%s" % ( scheme, host, port, diff --git a/consular/clients.py b/consular/clients.py new file mode 100644 index 0000000..a222813 --- /dev/null +++ b/consular/clients.py @@ -0,0 +1,104 @@ +from urllib import urlencode +import json + +from twisted.internet import reactor +from twisted.python import log +from twisted.web import client + +# Twisted's default HTTP11 client factory is way too verbose +client._HTTP11ClientFactory.noisy = False + +import treq + + +class JsonClient(object): + debug = False + clock = reactor + timeout = 5 + requester = lambda self, *a, **kw: treq.request(*a, **kw) + + def __init__(self): + self.pool = client.HTTPConnectionPool(self.clock, persistent=False) + + def _log_http_response(self, response, method, path, data): + log.msg('%s %s with %s returned: %s' % ( + method, path, data, response.code)) + return response + + def _log_http_error(self, failure, url): + log.err(failure, 'Error performing request to %s' % (url,)) + return failure + + def request(self, method, url, data, timeout=None): + d = self.requester( + method, + url.encode('utf-8'), + headers={ + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + data=(json.dumps(data) if data is not None else None), + pool=self.pool, + timeout=timeout or self.timeout) + + if self.debug: + d.addCallback(self._log_http_response, method, url, data) + + return d.addErrback(self._log_http_error, url) + + @classmethod + def response_json(cls, response): + return response.json() + + @classmethod + def response_ok(cls, response): + return response.code == 200 + + +class MarathonClient(JsonClient): + + def __init__(self, endpoint): + super(MarathonClient, self).__init__() + self.endpoint = endpoint + + def marathon_request(self, method, path, data=None): + return self.request(method, '%s%s' % (self.endpoint, path), data) + + def _basic_get_request(self, path, field, raise_error=True): + d = self.marathon_request('GET', path) + d.addErrback(log.err) + d.addCallback(JsonClient.response_json) + return d.addCallback(self._get_json_field, field, raise_error) + + def _get_json_field(self, response_json, field_name, raise_error=True): + if field_name not in response_json: + if raise_error: + raise KeyError('Unable to get value for "%s" from Marathon ' + 'response: "%s"' % ( + field_name, str(response_json),)) + else: + return None + + return response_json[field_name] + + def get_event_subscriptions(self): + return self._basic_get_request( + '/v2/eventSubscriptions', 'callbackUrls') + + def post_event_subscription(self, callback_url): + d = self.marathon_request( + 'POST', '/v2/eventSubscriptions?%s' % urlencode({ + 'callbackUrl': callback_url, + })) + d.addErrback(log.err) + return d.addCallback(JsonClient.response_ok) + + def get_apps(self): + return self._basic_get_request('/v2/apps', 'apps') + + def get_app(self, app_id): + return self._basic_get_request('/v2/apps%s' % (app_id,), 'app') + + def get_app_tasks(self, app_id, raise_error=True): + return self._basic_get_request( + '/v2/apps%s/tasks' % (app_id,), 'tasks', raise_error) diff --git a/consular/main.py b/consular/main.py index 608a19c..7013082 100644 --- a/consular/main.py +++ b/consular/main.py @@ -1,5 +1,7 @@ import json +from consular.clients import MarathonClient + from urllib import quote, urlencode from twisted.internet import reactor from twisted.web import client, server @@ -31,7 +33,7 @@ class ConsularSite(server.Site): debug = False def log(self, request): - if self.debug: + if self._debug: server.Site.log(self, request) @@ -51,16 +53,16 @@ class Consular(object): """ app = Klein() - debug = False + _debug = False clock = reactor - timeout = 5 + _timeout = 5 fallback_timeout = 2 - requester = lambda self, *a, **kw: treq.request(*a, **kw) + _requester = lambda self, *a, **kw: treq.request(*a, **kw) def __init__(self, consul_endpoint, marathon_endpoint, enable_fallback, registration_id): self.consul_endpoint = consul_endpoint - self.marathon_endpoint = marathon_endpoint + self.marathon_client = MarathonClient(marathon_endpoint) self.pool = client.HTTPConnectionPool(self.clock, persistent=False) self.enable_fallback = enable_fallback self.registration_id = registration_id @@ -68,6 +70,18 @@ def __init__(self, consul_endpoint, marathon_endpoint, enable_fallback, 'status_update_event': self.handle_status_update_event, } + def set_debug(self, debug): + self._debug = debug + self.marathon_client.debug = debug + + def set_timeout(self, timeout): + self._timeout = timeout + self.marathon_client.timeout = timeout + + def set_requester(self, requester): + self._requester = requester + self.marathon_client.requester = requester + def run(self, host, port): """ Starts the HTTP server. @@ -78,37 +92,9 @@ def run(self, host, port): The port to listen on (example is ``7000``) """ site = ConsularSite(self.app.resource()) - site.debug = self.debug + site.debug = self._debug self.clock.listenTCP(port, site, interface=host) - def get_marathon_event_callbacks(self): - d = self.marathon_request('GET', '/v2/eventSubscriptions') - d.addErrback(log.err) - d.addCallback(lambda response: response.json()) - d.addCallback(self.get_marathon_event_callbacks_from_json) - return d - - def get_marathon_event_callbacks_from_json(self, json): - # NOTE: - # Marathon may return a bad response when we get the existing event - # callbacks. A common cause for this is that Marathon is not properly - # configured. Raise an exception with information from Marathon if this - # is the case, else return the callback URLs from the JSON response. - if 'callbackUrls' not in json: - raise RuntimeError('Unable to get existing event callbacks from ' + - 'Marathon: %r' % (str(json),)) - - return json['callbackUrls'] - - def create_marathon_event_callback(self, url): - d = self.marathon_request( - 'POST', '/v2/eventSubscriptions?%s' % urlencode({ - 'callbackUrl': url, - })) - d.addErrback(log.err) - d.addCallback(lambda response: response.code == 200) - return d - @inlineCallbacks def register_marathon_event_callback(self, events_url): """ @@ -123,14 +109,16 @@ def register_marathon_event_callback(self, events_url): https://mesosphere.github.io/marathon/docs/event-bus.html #configuration """ - existing_callbacks = yield self.get_marathon_event_callbacks() + existing_callbacks = ( + yield self.marathon_client.get_event_subscriptions()) already_registered = any( [events_url == url for url in existing_callbacks]) if already_registered: log.msg('Consular event callback already registered.') returnValue(True) - registered = yield self.create_marathon_event_callback(events_url) + registered = ( + yield self.marathon_client.post_event_subscription(events_url)) if registered: log.msg('Consular event callback registered.') else: @@ -146,15 +134,11 @@ def log_http_error(self, failure, url): log.err(failure, 'Error performing request to %s' % (url,)) return failure - def marathon_request(self, method, path, data=None): - return self._request( - method, '%s%s' % (self.marathon_endpoint, path), data) - def consul_request(self, method, url, data=None): return self._request(method, url, data, timeout=self.fallback_timeout) def _request(self, method, url, data, timeout=None): - d = self.requester( + d = self._requester( method, url.encode('utf-8'), headers={ @@ -163,9 +147,9 @@ def _request(self, method, url, data, timeout=None): }, data=(json.dumps(data) if data is not None else None), pool=self.pool, - timeout=timeout or self.timeout) + timeout=timeout or self._timeout) - if self.debug: + if self._debug: d.addCallback(self.log_http_response, method, url, data) d.addErrback(self.log_http_error, url) @@ -229,7 +213,7 @@ def noop(self, request, event): def update_task_running(self, request, event): # NOTE: Marathon sends a list of ports, I don't know yet when & if # there are multiple values in that list. - d = self.get_app(event['appId']) + d = self.marathon_client.get_app(event['appId']) d.addCallback(lambda app: self.sync_app(app)) d.addCallback(lambda _: json.dumps({'status': 'ok'})) return d @@ -369,16 +353,10 @@ def sync_apps(self, purge=False): :param bool purge: To purge or not to purge. """ - d = self.get_marathon_apps() + d = self.marathon_client.get_apps() d.addCallback(self.check_apps_namespace_clash) return d.addCallback(self.sync_and_purge_apps, purge) - def get_marathon_apps(self): - """ Get a list of running apps from the Marathon API. """ - d = self.marathon_request('GET', '/v2/apps') - d.addCallback(lambda response: response.json()) - return d.addCallback(lambda data: data['apps']) - def sync_and_purge_apps(self, apps, purge=False): deferreds = [gatherResults([self.sync_app(app) for app in apps])] if purge: @@ -414,11 +392,6 @@ def check_apps_namespace_clash(self, apps): return apps - def get_app(self, app_id): - d = self.marathon_request('GET', '/v2/apps%s' % (app_id,)) - d = d.addCallback(lambda response: response.json()) - return d.addCallback(lambda data: data['app']) - def sync_app(self, app): return gatherResults([ self.sync_app_labels(app), @@ -535,10 +508,9 @@ def _consul_key_to_marathon_label_key(self, consul_key): return consul_key.split('/', 2)[-1] def sync_app_tasks(self, app): - d = self.marathon_request('GET', '/v2/apps%(id)s/tasks' % app) - d.addCallback(lambda response: response.json()) - return d.addCallback(lambda data: gatherResults( - self.sync_app_task(app, task) for task in data['tasks'])) + d = self.marathon_client.get_app_tasks(app['id']) + return d.addCallback(lambda tasks: gatherResults( + self.sync_app_task(app, task) for task in tasks)) def sync_app_task(self, app, task): return self.register_service( @@ -613,22 +585,30 @@ def purge_dead_agent_services(self, agent_endpoint): log.msg('Service "%s" does not have an app ID in its ' 'tags, it cannot be purged.' % (service['Service'],)) - elif self.debug: + elif self._debug: log.msg('Service "%s" is not tagged with our registration ID, ' 'not touching it.' % (service['Service'],)) for app_id, task_ids in services.items(): yield self.purge_service_if_dead(agent_endpoint, app_id, task_ids) - @inlineCallbacks def purge_service_if_dead(self, agent_endpoint, app_id, consul_task_ids): - response = yield self.marathon_request( - 'GET', '/v2/apps%s/tasks' % (app_id,)) - data = yield response.json() - tasks_to_be_purged = set(consul_task_ids) - if 'tasks' in data: - marathon_task_ids = set([task['id'] for task in data['tasks']]) - tasks_to_be_purged -= marathon_task_ids + # Get the running tasks for the app (don't raise an error if the tasks + # are not found) + d = self.marathon_client.get_app_tasks(app_id, raise_error=False) + + # Remove the running tasks from the set of Consul services + d.addCallback(self._filter_marathon_tasks, consul_task_ids) + + # Deregister the remaining old services + return d.addCallback(lambda service_ids: gatherResults( + [self.deregister_service(agent_endpoint, app_id, service_id) + for service_id in service_ids])) + + def _filter_marathon_tasks(self, marathon_tasks, consul_service_ids): + if not marathon_tasks: + return consul_service_ids - for task_id in tasks_to_be_purged: - yield self.deregister_service(agent_endpoint, app_id, task_id) + task_id_set = set([task['id'] for task in marathon_tasks]) + return [service_id for service_id in consul_service_ids + if service_id not in task_id_set] diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 4d6e849..2f7a1f4 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -45,7 +45,7 @@ def setUp(self): False, 'test' ) - self.consular.debug = True + self.consular.set_debug(True) # spin up a site so we can test it, pretty sure Klein has better # ways of doing this but they're not documented anywhere. @@ -71,7 +71,7 @@ def mock_requests(method, url, headers, data, pool, timeout): }) return d - self.patch(self.consular, 'requester', mock_requests) + self.consular.set_requester(mock_requests) def request(self, method, path, data=None): return treq.request( @@ -316,17 +316,17 @@ def test_register_with_marathon_unexpected_response(self): list_callbacks_request['deferred'].callback( FakeResponse(400, [], json.dumps({ 'message': - 'http event callback system is not running on this Marathon ' + - 'instance. Please re-start this instance with ' + + 'http event callback system is not running on this Marathon ' + 'instance. Please re-start this instance with ' '"--event_subscriber http_callback".'}))) - failure = self.failureResultOf(d, RuntimeError) + failure = self.failureResultOf(d, KeyError) self.assertEqual( failure.getErrorMessage(), - 'Unable to get existing event callbacks from Marathon: ' + - '\'{u\\\'message\\\': u\\\'http event callback system is not ' + - 'running on this Marathon instance. Please re-start this ' + - 'instance with "--event_subscriber http_callback".\\\'}\'') + '\'Unable to get value for "callbackUrls" from Marathon response: ' + '"{u\\\'message\\\': u\\\'http event callback system is not ' + 'running on this Marathon instance. Please re-start this instance ' + 'with "--event_subscriber http_callback".\\\'}"\'') @inlineCallbacks def test_sync_app_task(self): From 17f48192fa0a1a6a229ee807f20c01b3c5cc6b04 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Wed, 16 Sep 2015 17:48:45 +0200 Subject: [PATCH 033/111] Refactor Consul client into own class --- consular/clients.py | 68 +++++++++++++++++++++- consular/main.py | 112 ++++++------------------------------ consular/tests/test_main.py | 2 +- 3 files changed, 85 insertions(+), 97 deletions(-) diff --git a/consular/clients.py b/consular/clients.py index a222813..7f9f20e 100644 --- a/consular/clients.py +++ b/consular/clients.py @@ -1,4 +1,4 @@ -from urllib import urlencode +from urllib import quote, urlencode import json from twisted.internet import reactor @@ -102,3 +102,69 @@ def get_app(self, app_id): def get_app_tasks(self, app_id, raise_error=True): return self._basic_get_request( '/v2/apps%s/tasks' % (app_id,), 'tasks', raise_error) + + +class ConsulClient(JsonClient): + + fallback_timeout = 2 + + def __init__(self, endpoint, enable_fallback=False): + super(ConsulClient, self).__init__() + self.endpoint = endpoint + self.enable_fallback = enable_fallback + + def consul_request(self, method, url, data=None): + return self.request(method, url, data, timeout=self.fallback_timeout) + + def consul_local_request(self, method, path, data=None): + return self.consul_request( + method, '%s%s' % (self.endpoint, path,), data) + + def consul_agent_request(self, method, agent_endpoint, path, data=None): + return self.consul_request( + method, '%s%s' % (agent_endpoint, path,), data) + + def register_agent_service(self, agent_endpoint, registration): + d = self.consul_agent_request( + 'PUT', agent_endpoint, '/v1/agent/service/register', registration) + + if self.enable_fallback: + d.addErrback(self.register_agent_service_fallback, registration) + + return d + + def register_agent_service_fallback(self, failure, registration): + log.msg('Falling back for %s at %s.' % ( + registration['Name'], self.endpoint)) + return self.consul_local_request( + 'PUT', '/v1/agent/service/register', registration) + + def deregister_agent_service(self, agent_endpoint, service_id): + return self.consul_agent_request( + 'PUT', agent_endpoint, '/v1/agent/service/deregister/%s' % ( + service_id,)) + + def put_kv(self, key, value): + return self.consul_local_request( + 'PUT', '/v1/kv/%s' % (quote(key),), value) + + def get_kv_keys(self, keys_path, separator=None): + params = {'keys': ''} + if separator: + params['separator'] = separator + d = self.consul_local_request('GET', '/v1/kv/%s?%s' % ( + quote(keys_path), urlencode(params))) + return d.addCallback(JsonClient.response_json) + + def delete_kv_keys(self, key, recurse=False): + return self.consul_local_request('DELETE', '/v1/kv/%s%s' % ( + quote(key), '?recurse' if recurse else '',)) + + def get_catalog_nodes(self): + d = self.consul_local_request('GET', '/v1/catalog/nodes') + return d.addCallback(JsonClient.response_json) + + def get_agent_services(self, agent_endpoint): + d = self.consul_agent_request( + 'GET', agent_endpoint, '/v1/agent/services') + return d.addCallback(JsonClient.response_json) diff --git a/consular/main.py b/consular/main.py index 7013082..eadf356 100644 --- a/consular/main.py +++ b/consular/main.py @@ -1,18 +1,13 @@ import json -from consular.clients import MarathonClient +from consular.clients import ConsulClient, MarathonClient -from urllib import quote, urlencode from twisted.internet import reactor -from twisted.web import client, server -# Twisted's fault HTTP11 client factory is way too verbose -client._HTTP11ClientFactory.noisy = False +from twisted.web import server from twisted.internet.defer import ( succeed, inlineCallbacks, returnValue, gatherResults) from twisted.python import log - -import treq from klein import Klein @@ -55,16 +50,11 @@ class Consular(object): app = Klein() _debug = False clock = reactor - _timeout = 5 - fallback_timeout = 2 - _requester = lambda self, *a, **kw: treq.request(*a, **kw) def __init__(self, consul_endpoint, marathon_endpoint, enable_fallback, registration_id): - self.consul_endpoint = consul_endpoint + self.consul_client = ConsulClient(consul_endpoint, enable_fallback) self.marathon_client = MarathonClient(marathon_endpoint) - self.pool = client.HTTPConnectionPool(self.clock, persistent=False) - self.enable_fallback = enable_fallback self.registration_id = registration_id self.event_dispatch = { 'status_update_event': self.handle_status_update_event, @@ -72,14 +62,15 @@ def __init__(self, consul_endpoint, marathon_endpoint, enable_fallback, def set_debug(self, debug): self._debug = debug + self.consul_client.debug = debug self.marathon_client.debug = debug def set_timeout(self, timeout): - self._timeout = timeout + self.consul_client.timeout = timeout self.marathon_client.timeout = timeout def set_requester(self, requester): - self._requester = requester + self.consul_client.requester = requester self.marathon_client.requester = requester def run(self, host, port): @@ -125,36 +116,6 @@ def register_marathon_event_callback(self, events_url): log.err('Consular event callback registration failed.') returnValue(registered) - def log_http_response(self, response, method, path, data): - log.msg('%s %s with %s returned: %s' % ( - method, path, data, response.code)) - return response - - def log_http_error(self, failure, url): - log.err(failure, 'Error performing request to %s' % (url,)) - return failure - - def consul_request(self, method, url, data=None): - return self._request(method, url, data, timeout=self.fallback_timeout) - - def _request(self, method, url, data, timeout=None): - d = self._requester( - method, - url.encode('utf-8'), - headers={ - 'Content-Type': 'application/json', - 'Accept': 'application/json', - }, - data=(json.dumps(data) if data is not None else None), - pool=self.pool, - timeout=timeout or self._timeout) - - if self._debug: - d.addCallback(self.log_http_response, method, url, data) - - d.addErrback(self.log_http_error, url) - return d - @app.route('/') def index(self, request): request.setHeader('Content-Type', 'application/json') @@ -307,21 +268,8 @@ def register_service(self, agent_endpoint, registration = self._create_service_registration(app_id, service_id, address, port) - d = self.consul_request( - 'PUT', - '%s/v1/agent/service/register' % (agent_endpoint,), - registration) - if self.enable_fallback: - d.addErrback(self.register_service_fallback, registration) - return d - - def register_service_fallback(self, failure, registration): - log.msg('Falling back for %s at %s.' % ( - registration['Name'], self.consul_endpoint)) - return self.consul_request( - 'PUT', - '%s/v1/agent/service/register' % (self.consul_endpoint,), - registration) + return self.consul_client.register_agent_service( + agent_endpoint, registration) def deregister_service(self, agent_endpoint, app_id, service_id): """ @@ -337,9 +285,8 @@ def deregister_service(self, agent_endpoint, app_id, service_id): """ log.msg('Deregistering %s at %s with %s' % ( app_id, agent_endpoint, service_id,)) - return self.consul_request( - 'PUT', '%s/v1/agent/service/deregister/%s' % ( - agent_endpoint, service_id,)) + return self.consul_client.deregister_agent_service( + agent_endpoint, service_id) def sync_apps(self, purge=False): """ @@ -432,14 +379,9 @@ def put_consul_app_labels(self, app_name, labels): def put_consul_kvs(self, key_values): """ Store the given key/value set in the Consul k/v store. """ - return gatherResults([self.put_consul_kv(key, value) + return gatherResults([self.consul_client.put_kv(key, value) for key, value in key_values.items()]) - def put_consul_kv(self, key, value): - """ Store the given value at the given key in the Consul k/v store. """ - return self.consul_request('PUT', '%s/v1/kv/%s' % ( - self.consul_endpoint, quote(key),), value) - def clean_consul_app_labels(self, app_name, labels): """ Delete app labels stored in the Consul k/v store under the given app @@ -456,35 +398,20 @@ def clean_consul_app_labels(self, app_name, labels): def get_consul_app_keys(self, app_name): """ Get the Consul k/v keys for the app with the given name. """ - return self.get_consul_kv_keys('consular/%s' % (app_name,)) + return self.consul_client.get_kv_keys('consular/%s' % (app_name,)) def get_consul_consular_keys(self): """ Get the next level of Consul k/v keys at 'consular/', i.e. will return 'consular/my-app' but not 'consular/my-app/my-label'. """ - return self.get_consul_kv_keys('consular/', separator='/') - - def get_consul_kv_keys(self, key_path, separator=None): - """ Get the Consul k/v keys present at the given key path. """ - params = {'keys': ''} - if separator: - params['separator'] = separator - d = self.consul_request('GET', '%s/v1/kv/%s?%s' % ( - self.consul_endpoint, quote(key_path), urlencode(params))) - return d.addCallback(lambda response: response.json()) + return self.consul_client.get_kv_keys('consular/', separator='/') def delete_consul_kv_keys(self, keys, recurse=False): """ Delete a sequence of Consul k/v keys. """ - return gatherResults([self.delete_consul_kv_key(key, recurse) + return gatherResults([self.consul_client.delete_kv_keys(key, recurse) for key in keys]) - def delete_consul_kv_key(self, key, recurse=False): - """ Delete the Consul k/v entry associated with the given key. """ - return self.consul_request('DELETE', '%s/v1/kv/%s%s' % ( - self.consul_endpoint, quote(key), - '?recurse' if recurse else '',)) - def _filter_marathon_labels(self, consul_keys, marathon_labels): """ Takes a list of Consul keys and removes those with keys not found in @@ -557,20 +484,15 @@ def _consul_key_to_marathon_app_name(self, consul_key): return consul_key.split('/', 1)[-1] def purge_dead_services(self): - d = self.consul_request( - 'GET', '%s/v1/catalog/nodes' % (self.consul_endpoint,)) - d.addCallback(lambda response: response.json()) - d.addCallback(lambda data: gatherResults([ + d = self.consul_client.get_catalog_nodes() + return d.addCallback(lambda data: gatherResults([ self.purge_dead_agent_services( get_agent_endpoint(node['Address'])) for node in data ])) - return d @inlineCallbacks def purge_dead_agent_services(self, agent_endpoint): - response = yield self.consul_request( - 'GET', '%s/v1/agent/services' % (agent_endpoint,)) - data = yield response.json() + data = yield self.consul_client.get_agent_services(agent_endpoint) # collect the task ids for the service name services = {} diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 2f7a1f4..c27ea0b 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -760,7 +760,7 @@ def test_purge_dead_app_labels(self): @inlineCallbacks def test_fallback_to_main_consul(self): - self.consular.enable_fallback = True + self.consular.consul_client.enable_fallback = True self.consular.register_service( 'http://foo:8500', '/app_id', 'service_id', 'foo', 1234) request = yield self.requests.get() From 517f1719889b15c4c49466e2d1efd9f9d2b04c75 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Wed, 16 Sep 2015 17:52:00 +0200 Subject: [PATCH 034/111] Log client/http errors in one place * Let JsonClient._log_http_error handle everything --- consular/clients.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/consular/clients.py b/consular/clients.py index 7f9f20e..4bb7f7e 100644 --- a/consular/clients.py +++ b/consular/clients.py @@ -66,7 +66,6 @@ def marathon_request(self, method, path, data=None): def _basic_get_request(self, path, field, raise_error=True): d = self.marathon_request('GET', path) - d.addErrback(log.err) d.addCallback(JsonClient.response_json) return d.addCallback(self._get_json_field, field, raise_error) @@ -90,7 +89,6 @@ def post_event_subscription(self, callback_url): 'POST', '/v2/eventSubscriptions?%s' % urlencode({ 'callbackUrl': callback_url, })) - d.addErrback(log.err) return d.addCallback(JsonClient.response_ok) def get_apps(self): From 0042936578a9c20e3406c8fb8b861d9f08bcf227 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Wed, 16 Sep 2015 18:08:11 +0200 Subject: [PATCH 035/111] Clean up Consul request method --- consular/clients.py | 47 ++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/consular/clients.py b/consular/clients.py index 4bb7f7e..7b3e3d6 100644 --- a/consular/clients.py +++ b/consular/clients.py @@ -29,7 +29,7 @@ def _log_http_error(self, failure, url): log.err(failure, 'Error performing request to %s' % (url,)) return failure - def request(self, method, url, data, timeout=None): + def request(self, method, url, data=None, timeout=None): d = self.requester( method, url.encode('utf-8'), @@ -111,20 +111,19 @@ def __init__(self, endpoint, enable_fallback=False): self.endpoint = endpoint self.enable_fallback = enable_fallback - def consul_request(self, method, url, data=None): - return self.request(method, url, data, timeout=self.fallback_timeout) + def consul_request(self, method, path, endpoint=None, data=None, + timeout=None): + if not endpoint: + endpoint = self.endpoint + if not timeout: + timeout = self.timeout - def consul_local_request(self, method, path, data=None): - return self.consul_request( - method, '%s%s' % (self.endpoint, path,), data) - - def consul_agent_request(self, method, agent_endpoint, path, data=None): - return self.consul_request( - method, '%s%s' % (agent_endpoint, path,), data) + return self.request( + method, '%s%s' % (endpoint, path,), data=data, timeout=timeout) def register_agent_service(self, agent_endpoint, registration): - d = self.consul_agent_request( - 'PUT', agent_endpoint, '/v1/agent/service/register', registration) + d = self.consul_request('PUT', '/v1/agent/service/register', + endpoint=agent_endpoint, data=registration) if self.enable_fallback: d.addErrback(self.register_agent_service_fallback, registration) @@ -134,35 +133,35 @@ def register_agent_service(self, agent_endpoint, registration): def register_agent_service_fallback(self, failure, registration): log.msg('Falling back for %s at %s.' % ( registration['Name'], self.endpoint)) - return self.consul_local_request( - 'PUT', '/v1/agent/service/register', registration) + return self.consul_request( + 'PUT', '/v1/agent/service/register', data=registration, + timeout=self.fallback_timeout) def deregister_agent_service(self, agent_endpoint, service_id): - return self.consul_agent_request( - 'PUT', agent_endpoint, '/v1/agent/service/deregister/%s' % ( - service_id,)) + return self.consul_request('PUT', '/v1/agent/service/deregister/%s' % ( + service_id,), endpoint=agent_endpoint) def put_kv(self, key, value): - return self.consul_local_request( - 'PUT', '/v1/kv/%s' % (quote(key),), value) + return self.consul_request( + 'PUT', '/v1/kv/%s' % (quote(key),), data=value) def get_kv_keys(self, keys_path, separator=None): params = {'keys': ''} if separator: params['separator'] = separator - d = self.consul_local_request('GET', '/v1/kv/%s?%s' % ( + d = self.consul_request('GET', '/v1/kv/%s?%s' % ( quote(keys_path), urlencode(params))) return d.addCallback(JsonClient.response_json) def delete_kv_keys(self, key, recurse=False): - return self.consul_local_request('DELETE', '/v1/kv/%s%s' % ( + return self.consul_request('DELETE', '/v1/kv/%s%s' % ( quote(key), '?recurse' if recurse else '',)) def get_catalog_nodes(self): - d = self.consul_local_request('GET', '/v1/catalog/nodes') + d = self.consul_request('GET', '/v1/catalog/nodes') return d.addCallback(JsonClient.response_json) def get_agent_services(self, agent_endpoint): - d = self.consul_agent_request( - 'GET', agent_endpoint, '/v1/agent/services') + d = self.consul_request( + 'GET', '/v1/agent/services', endpoint=agent_endpoint) return d.addCallback(JsonClient.response_json) From 1e076672b6fb3b2fc90eb9c86c04fdbbd72f4436 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Thu, 17 Sep 2015 14:11:44 +0200 Subject: [PATCH 036/111] Further cleanup of HTTP clients * Keep a default endpoint in the base class, add a kwarg for specifying a different endpoint. * Add get_json() method to perform a GET request expecting a JSON response. --- consular/clients.py | 111 ++++++++++++++++++------------------ consular/tests/test_main.py | 4 +- 2 files changed, 59 insertions(+), 56 deletions(-) diff --git a/consular/clients.py b/consular/clients.py index 7b3e3d6..01b5f64 100644 --- a/consular/clients.py +++ b/consular/clients.py @@ -17,7 +17,8 @@ class JsonClient(object): timeout = 5 requester = lambda self, *a, **kw: treq.request(*a, **kw) - def __init__(self): + def __init__(self, endpoint): + self.endpoint = endpoint self.pool = client.HTTPConnectionPool(self.clock, persistent=False) def _log_http_response(self, response, method, path, data): @@ -29,44 +30,60 @@ def _log_http_error(self, failure, url): log.err(failure, 'Error performing request to %s' % (url,)) return failure - def request(self, method, url, data=None, timeout=None): - d = self.requester( - method, - url.encode('utf-8'), - headers={ + def request(self, method, path, endpoint=None, json_data=None, **kwargs): + """ + Perform a request. A number of basic defaults are set on the request + that make using a JSON API easier. + + :param: method: + The HTTP method to use (example is `GET`). + :param: path: + The URL path. This is appended to the endpoint and should start + with a '/' (example is `/v2/apps`). + :param: endpoint: + The URL endpoint to use. The default value is the endpoint this + client was created with (`self.endpoint`) (example is + `http://localhost:8080`) + :param: json_data: + A python data structure that will be converted to a JSON string + using `json.dumps` and used as the request body. + :param: kwargs: + Any other parameters that will be passed to `treq.request`, for + example headers or parameters. + """ + url = ('%s%s' % (endpoint or self.endpoint, path)).encode('utf-8') + + data = json.dumps(json_data) if json_data else None + requester_kwargs = { + 'headers': { 'Content-Type': 'application/json', 'Accept': 'application/json', }, - data=(json.dumps(data) if data is not None else None), - pool=self.pool, - timeout=timeout or self.timeout) + 'data': data, + 'pool': self.pool, + 'timeout': self.timeout + } + requester_kwargs.update(kwargs) + + d = self.requester(method, url, **requester_kwargs) if self.debug: d.addCallback(self._log_http_response, method, url, data) return d.addErrback(self._log_http_error, url) - @classmethod - def response_json(cls, response): - return response.json() - - @classmethod - def response_ok(cls, response): - return response.code == 200 + def get_json(self, path, **kwargs): + """ + Perform a GET request to the given path and return the JSON response. + """ + d = self.request('GET', path, **kwargs) + return d.addCallback(lambda response: response.json()) class MarathonClient(JsonClient): - def __init__(self, endpoint): - super(MarathonClient, self).__init__() - self.endpoint = endpoint - - def marathon_request(self, method, path, data=None): - return self.request(method, '%s%s' % (self.endpoint, path), data) - def _basic_get_request(self, path, field, raise_error=True): - d = self.marathon_request('GET', path) - d.addCallback(JsonClient.response_json) + d = self.get_json(path) return d.addCallback(self._get_json_field, field, raise_error) def _get_json_field(self, response_json, field_name, raise_error=True): @@ -85,11 +102,11 @@ def get_event_subscriptions(self): '/v2/eventSubscriptions', 'callbackUrls') def post_event_subscription(self, callback_url): - d = self.marathon_request( + d = self.request( 'POST', '/v2/eventSubscriptions?%s' % urlencode({ 'callbackUrl': callback_url, })) - return d.addCallback(JsonClient.response_ok) + return d.addCallback(lambda response: response.code == 200) def get_apps(self): return self._basic_get_request('/v2/apps', 'apps') @@ -107,23 +124,13 @@ class ConsulClient(JsonClient): fallback_timeout = 2 def __init__(self, endpoint, enable_fallback=False): - super(ConsulClient, self).__init__() + super(ConsulClient, self).__init__(endpoint) self.endpoint = endpoint self.enable_fallback = enable_fallback - def consul_request(self, method, path, endpoint=None, data=None, - timeout=None): - if not endpoint: - endpoint = self.endpoint - if not timeout: - timeout = self.timeout - - return self.request( - method, '%s%s' % (endpoint, path,), data=data, timeout=timeout) - def register_agent_service(self, agent_endpoint, registration): - d = self.consul_request('PUT', '/v1/agent/service/register', - endpoint=agent_endpoint, data=registration) + d = self.request('PUT', '/v1/agent/service/register', + endpoint=agent_endpoint, json_data=registration) if self.enable_fallback: d.addErrback(self.register_agent_service_fallback, registration) @@ -133,35 +140,31 @@ def register_agent_service(self, agent_endpoint, registration): def register_agent_service_fallback(self, failure, registration): log.msg('Falling back for %s at %s.' % ( registration['Name'], self.endpoint)) - return self.consul_request( - 'PUT', '/v1/agent/service/register', data=registration, + return self.request( + 'PUT', '/v1/agent/service/register', json_data=registration, timeout=self.fallback_timeout) def deregister_agent_service(self, agent_endpoint, service_id): - return self.consul_request('PUT', '/v1/agent/service/deregister/%s' % ( + return self.request('PUT', '/v1/agent/service/deregister/%s' % ( service_id,), endpoint=agent_endpoint) def put_kv(self, key, value): - return self.consul_request( - 'PUT', '/v1/kv/%s' % (quote(key),), data=value) + return self.request( + 'PUT', '/v1/kv/%s' % (quote(key),), json_data=value) def get_kv_keys(self, keys_path, separator=None): params = {'keys': ''} if separator: params['separator'] = separator - d = self.consul_request('GET', '/v1/kv/%s?%s' % ( - quote(keys_path), urlencode(params))) - return d.addCallback(JsonClient.response_json) + return self.get_json('/v1/kv/%s?%s' % (quote(keys_path), + urlencode(params))) def delete_kv_keys(self, key, recurse=False): - return self.consul_request('DELETE', '/v1/kv/%s%s' % ( + return self.request('DELETE', '/v1/kv/%s%s' % ( quote(key), '?recurse' if recurse else '',)) def get_catalog_nodes(self): - d = self.consul_request('GET', '/v1/catalog/nodes') - return d.addCallback(JsonClient.response_json) + return self.get_json('/v1/catalog/nodes') def get_agent_services(self, agent_endpoint): - d = self.consul_request( - 'GET', '/v1/agent/services', endpoint=agent_endpoint) - return d.addCallback(JsonClient.response_json) + return self.get_json('/v1/agent/services', endpoint=agent_endpoint) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index c27ea0b..f77750a 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -61,12 +61,12 @@ def setUp(self): # We use this to mock requests going to Consul & Marathon self.requests = DeferredQueue() - def mock_requests(method, url, headers, data, pool, timeout): + def mock_requests(method, url, **kwargs): d = Deferred() self.requests.put({ 'method': method, 'url': url, - 'data': data, + 'data': kwargs.get('data'), 'deferred': d, }) return d From f2c66e67e1485b4753a3d272e60b6dcfcbd50489 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Thu, 17 Sep 2015 14:59:49 +0200 Subject: [PATCH 037/111] Add comments to new classes --- consular/clients.py | 87 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/consular/clients.py b/consular/clients.py index 01b5f64..a18a9f7 100644 --- a/consular/clients.py +++ b/consular/clients.py @@ -18,6 +18,9 @@ class JsonClient(object): requester = lambda self, *a, **kw: treq.request(*a, **kw) def __init__(self, endpoint): + """ + Create a client with the specified default endpoint. + """ self.endpoint = endpoint self.pool = client.HTTPConnectionPool(self.clock, persistent=False) @@ -83,10 +86,29 @@ def get_json(self, path, **kwargs): class MarathonClient(JsonClient): def _basic_get_request(self, path, field, raise_error=True): + """ + Perform a GET request and get the contents of the JSON response. + + Marathon's JSON responses tend to contain an object with a single key + which points to the actual data of the response. For example /v2/apps + returns something like {"apps": [ {"app1"}, {"app2"} ]}. We're + interested in the contents of "apps". + """ d = self.get_json(path) return d.addCallback(self._get_json_field, field, raise_error) def _get_json_field(self, response_json, field_name, raise_error=True): + """ + Get a JSON field from the response JSON. + + :param: response_json: + The parsed JSON content of the response. + :param: field_name: + The name of the field in the JSON to get. + :param: raise_error: + If True, an error will be raised if the field is not present in + the response JSON. If False, None is returned. + """ if field_name not in response_json: if raise_error: raise KeyError('Unable to get value for "%s" from Marathon ' @@ -98,10 +120,17 @@ def _get_json_field(self, response_json, field_name, raise_error=True): return response_json[field_name] def get_event_subscriptions(self): + """ + Get the current Marathon event subscriptions, returning a list of + callback URLs. + """ return self._basic_get_request( '/v2/eventSubscriptions', 'callbackUrls') def post_event_subscription(self, callback_url): + """ + Post a new Marathon event subscription with the given callback URL. + """ d = self.request( 'POST', '/v2/eventSubscriptions?%s' % urlencode({ 'callbackUrl': callback_url, @@ -109,12 +138,23 @@ def post_event_subscription(self, callback_url): return d.addCallback(lambda response: response.code == 200) def get_apps(self): + """ + Get the currently running Marathon apps, returning a list of app + definitions. + """ return self._basic_get_request('/v2/apps', 'apps') def get_app(self, app_id): + """ + Get information about the app with the given app ID. + """ return self._basic_get_request('/v2/apps%s' % (app_id,), 'app') def get_app_tasks(self, app_id, raise_error=True): + """ + Get the currently running tasks for the app with the given app ID, + returning a list of task definitions. + """ return self._basic_get_request( '/v2/apps%s/tasks' % (app_id,), 'tasks', raise_error) @@ -124,11 +164,24 @@ class ConsulClient(JsonClient): fallback_timeout = 2 def __init__(self, endpoint, enable_fallback=False): + """ + Create a Consul client. + + :param: endpoint: + The default Consul endpoint, usually on the same node as Consular + is running. + :param: enable_fallback: + Fall back to the default Consul endpoint when registering services + on an agent that cannot be reached. + """ super(ConsulClient, self).__init__(endpoint) self.endpoint = endpoint self.enable_fallback = enable_fallback def register_agent_service(self, agent_endpoint, registration): + """ + Register a Consul service at the given agent endpoint. + """ d = self.request('PUT', '/v1/agent/service/register', endpoint=agent_endpoint, json_data=registration) @@ -138,6 +191,10 @@ def register_agent_service(self, agent_endpoint, registration): return d def register_agent_service_fallback(self, failure, registration): + """ + Fallback to the default agent endpoint (`self.endpoint`) to register + a Consul service. + """ log.msg('Falling back for %s at %s.' % ( registration['Name'], self.endpoint)) return self.request( @@ -145,14 +202,30 @@ def register_agent_service_fallback(self, failure, registration): timeout=self.fallback_timeout) def deregister_agent_service(self, agent_endpoint, service_id): + """ + Deregister a Consul service at the given agent endpoint. + """ return self.request('PUT', '/v1/agent/service/deregister/%s' % ( service_id,), endpoint=agent_endpoint) def put_kv(self, key, value): + """ + Put a key/value in Consul's k/v store. + """ return self.request( 'PUT', '/v1/kv/%s' % (quote(key),), json_data=value) def get_kv_keys(self, keys_path, separator=None): + """ + Get the stored keys for the given keys path from the Consul k/v store. + + :param: keys_path: + The path to some keys (example is `consular/my-app/`). + :param: separator: + Get all the keys up to some separator in the key path. Useful for + getting all the keys non-recursively for a path. For more + information see the Consul API documentation. + """ params = {'keys': ''} if separator: params['separator'] = separator @@ -160,11 +233,25 @@ def get_kv_keys(self, keys_path, separator=None): urlencode(params))) def delete_kv_keys(self, key, recurse=False): + """ + Delete the store key(s) at the given path from the Consul k/v store. + + :param: key: + The key or key path to be deleted. + :param: recurse: + Whether or not to recursively delete all subpaths of the key. + """ return self.request('DELETE', '/v1/kv/%s%s' % ( quote(key), '?recurse' if recurse else '',)) def get_catalog_nodes(self): + """ + Get the list of active Consul nodes from the catalog. + """ return self.get_json('/v1/catalog/nodes') def get_agent_services(self, agent_endpoint): + """ + Get the list of running services for the given agent endpoint. + """ return self.get_json('/v1/agent/services', endpoint=agent_endpoint) From 6856968f530b35aadfdccb3b099c657fedf5cf92 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Thu, 17 Sep 2015 18:13:31 +0200 Subject: [PATCH 038/111] Add failing tests --- consular/tests/test_main.py | 47 +++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 4d6e849..689d414 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -453,6 +453,33 @@ def test_sync_app_labels_cleanup(self): yield d + @inlineCallbacks + def test_sync_app_labels_cleanup_not_found(self): + """ + When Consular syncs app labels, and labels aren't found in Consul and + Consul returns a 404, we should fail gracefully. + """ + app = { + 'id': '/my-app', + 'labels': {'foo': 'bar'} + } + d = self.consular.sync_app_labels(app) + put_request = yield self.requests.get() + self.assertEqual(put_request['method'], 'PUT') + self.assertEqual(put_request['url'], + 'http://localhost:8500/v1/kv/consular/my-app/foo') + self.assertEqual(put_request['data'], '"bar"') + put_request['deferred'].callback( + FakeResponse(200, [], json.dumps({}))) + + get_request = yield self.requests.get() + self.assertEqual(get_request['method'], 'GET') + self.assertEqual(get_request['url'], + 'http://localhost:8500/v1/kv/consular/my-app?keys=') + get_request['deferred'].callback(FakeResponse(404, [], None)) + + yield d + @inlineCallbacks def test_sync_app(self): app = { @@ -758,6 +785,26 @@ def test_purge_dead_app_labels(self): yield d + @inlineCallbacks + def test_purge_dead_app_labels_not_found(self): + """ + When purging labels from the Consul k/v store, if Consul can't find + a key and returns 404, we should fail gracefully. + """ + d = self.consular.purge_dead_app_labels([{ + 'id': 'my-app' + }]) + consul_request = yield self.requests.get() + self.assertEqual(consul_request['method'], 'GET') + self.assertEqual( + consul_request['url'], + 'http://localhost:8500/v1/kv/consular/?keys=&separator=%2F') + # Return one existing app and one non-existing app + consul_request['deferred'].callback(FakeResponse(404, [], None)) + + # No keys exist in Consul so nothing to purge + yield d + @inlineCallbacks def test_fallback_to_main_consul(self): self.consular.enable_fallback = True From a9b37e58f5b10162941318bcdad2d928609d3bba Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Thu, 17 Sep 2015 18:37:09 +0200 Subject: [PATCH 039/111] Check response code from Consul when getting keys --- consular/main.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/consular/main.py b/consular/main.py index 608a19c..98b7332 100644 --- a/consular/main.py +++ b/consular/main.py @@ -499,7 +499,25 @@ def get_consul_kv_keys(self, key_path, separator=None): params['separator'] = separator d = self.consul_request('GET', '%s/v1/kv/%s?%s' % ( self.consul_endpoint, quote(key_path), urlencode(params))) - return d.addCallback(lambda response: response.json()) + return d.addCallback(self._get_consul_kv_keys_response, key_path) + + def _get_consul_kv_keys_response(self, response, key_path): + """ + Get the list of keys from a Consul k/v store response. If the keys were + not found (Consul returned a 404), return an empty list. + """ + if response.code == 200: + return response.json() + elif response.code == 404: + if self.debug: + log.msg( + 'Consul returned a 404 when getting the keys for %s' % ( + key_path,)) + return [] + else: + raise RuntimeError('Unexpected response from Consul when getting ' + 'keys for "%s", status code = %s' % ( + key_path, response.code,)) def delete_consul_kv_keys(self, keys, recurse=False): """ Delete a sequence of Consul k/v keys. """ From fadf8401e83ff17b9dec20982a1c95ba0ace376d Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Fri, 18 Sep 2015 09:39:45 +0200 Subject: [PATCH 040/111] Thou shalt not decrease test coverage * Add tests for unexpected responses from Consul --- consular/tests/test_main.py | 60 ++++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 689d414..44e3c71 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -5,7 +5,7 @@ from twisted.web.server import Site from twisted.internet import reactor from twisted.internet.defer import ( - inlineCallbacks, DeferredQueue, Deferred, succeed) + inlineCallbacks, DeferredQueue, Deferred, FirstError, succeed) from twisted.web.client import HTTPConnectionPool from twisted.python import log @@ -480,6 +480,39 @@ def test_sync_app_labels_cleanup_not_found(self): yield d + @inlineCallbacks + def test_sync_app_labels_cleanup_forbidden(self): + """ + When Consular syncs app labels, and labels aren't found in Consul and + Consul returns a 403, an error should be raised. + """ + app = { + 'id': '/my-app', + 'labels': {'foo': 'bar'} + } + d = self.consular.sync_app_labels(app) + put_request = yield self.requests.get() + self.assertEqual(put_request['method'], 'PUT') + self.assertEqual(put_request['url'], + 'http://localhost:8500/v1/kv/consular/my-app/foo') + self.assertEqual(put_request['data'], '"bar"') + put_request['deferred'].callback( + FakeResponse(200, [], json.dumps({}))) + + get_request = yield self.requests.get() + self.assertEqual(get_request['method'], 'GET') + self.assertEqual(get_request['url'], + 'http://localhost:8500/v1/kv/consular/my-app?keys=') + get_request['deferred'].callback(FakeResponse(403, [], None)) + + failure = self.failureResultOf(d, FirstError) + error = failure.value.subFailure.value + self.assertIsInstance(error, RuntimeError) + self.assertEqual( + str(error), + 'Unexpected response from Consul when getting keys for ' + '"consular/my-app", status code = 403') + @inlineCallbacks def test_sync_app(self): app = { @@ -805,6 +838,31 @@ def test_purge_dead_app_labels_not_found(self): # No keys exist in Consul so nothing to purge yield d + @inlineCallbacks + def test_purge_dead_app_labels_forbidden(self): + """ + When purging labels from the Consul k/v store, if Consul can't find + a key and returns 403, an error should be raised. + """ + d = self.consular.purge_dead_app_labels([{ + 'id': 'my-app' + }]) + consul_request = yield self.requests.get() + self.assertEqual(consul_request['method'], 'GET') + self.assertEqual( + consul_request['url'], + 'http://localhost:8500/v1/kv/consular/?keys=&separator=%2F') + # Return one existing app and one non-existing app + consul_request['deferred'].callback(FakeResponse(403, [], None)) + + failure = self.failureResultOf(d, RuntimeError) + error = failure.value + self.assertIsInstance(error, RuntimeError) + self.assertEqual( + str(error), + 'Unexpected response from Consul when getting keys for ' + '"consular/", status code = 403') + @inlineCallbacks def test_fallback_to_main_consul(self): self.consular.enable_fallback = True From 973af6067e64b60d248ca6874bb3818616a07920 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Fri, 18 Sep 2015 09:48:38 +0200 Subject: [PATCH 041/111] Clean up new tests --- consular/tests/test_main.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 44e3c71..9fc4fb3 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -505,11 +505,12 @@ def test_sync_app_labels_cleanup_forbidden(self): 'http://localhost:8500/v1/kv/consular/my-app?keys=') get_request['deferred'].callback(FakeResponse(403, [], None)) + # Error is raised into a DeferredList, must get actual error failure = self.failureResultOf(d, FirstError) - error = failure.value.subFailure.value - self.assertIsInstance(error, RuntimeError) + actual_failure = failure.value.subFailure + self.assertEqual(actual_failure.type, RuntimeError) self.assertEqual( - str(error), + actual_failure.getErrorMessage(), 'Unexpected response from Consul when getting keys for ' '"consular/my-app", status code = 403') @@ -832,7 +833,7 @@ def test_purge_dead_app_labels_not_found(self): self.assertEqual( consul_request['url'], 'http://localhost:8500/v1/kv/consular/?keys=&separator=%2F') - # Return one existing app and one non-existing app + # Return a 404 error consul_request['deferred'].callback(FakeResponse(404, [], None)) # No keys exist in Consul so nothing to purge @@ -852,14 +853,12 @@ def test_purge_dead_app_labels_forbidden(self): self.assertEqual( consul_request['url'], 'http://localhost:8500/v1/kv/consular/?keys=&separator=%2F') - # Return one existing app and one non-existing app + # Return a 403 error consul_request['deferred'].callback(FakeResponse(403, [], None)) failure = self.failureResultOf(d, RuntimeError) - error = failure.value - self.assertIsInstance(error, RuntimeError) self.assertEqual( - str(error), + failure.getErrorMessage(), 'Unexpected response from Consul when getting keys for ' '"consular/", status code = 403') From 9ee9830f2b6bba45a9a7d0140925270cede69d77 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Fri, 18 Sep 2015 14:42:18 +0200 Subject: [PATCH 042/111] Rework handling of unexpected responses * Only try get JSON content if OK response code * Add new UnexpectedResponseError type * Handle 404's that should be ignored in main Consular class * Add tests --- consular/clients.py | 90 ++++++++++++++------------ consular/main.py | 25 +++++++- consular/tests/test_main.py | 123 ++++++++++++++++++++++++++++++++---- 3 files changed, 184 insertions(+), 54 deletions(-) diff --git a/consular/clients.py b/consular/clients.py index 476d42e..e494816 100644 --- a/consular/clients.py +++ b/consular/clients.py @@ -4,6 +4,7 @@ from twisted.internet import reactor from twisted.python import log from twisted.web import client +from twisted.web.http import OK # Twisted's default HTTP11 client factory is way too verbose client._HTTP11ClientFactory.noisy = False @@ -80,12 +81,48 @@ def get_json(self, path, **kwargs): Perform a GET request to the given path and return the JSON response. """ d = self.request('GET', path, **kwargs) - return d.addCallback(lambda response: response.json()) + return d.addCallback(self._response_json_if_ok) + + def _response_json_if_ok(self, response): + """ + Get the response JSON content if the respones code is OK (200), else + raise an `UnexpectedResponseError`. + """ + if response.code == OK: + return response.json() + else: + d = response.content() + d.addCallback(self._raise_unexpected_response_error, response) + return d + + def _raise_unexpected_response_error(self, response_content, response): + raise UnexpectedResponseError(response, response_content) + + +class UnexpectedResponseError(Exception): + """ + Error raised for a non-200 response code. + """ + def __init__(self, response, response_content, message=None): + if not message: + message = self._default_error_message(response, response_content) + + super(UnexpectedResponseError, self).__init__(message) + self.response = response + + def _default_error_message(self, response, response_content): + # Due to current testing method we can't get the Twisted Request object + # from the response and add extra useful fields to the error message. + + # request = response.request + return 'response: code=%d, body=%s \nrequest: method=, url=, body=' % ( + response.code, response_content,) + # request.method, request.url, request.data)) class MarathonClient(JsonClient): - def _basic_get_request(self, path, field, raise_error=True): + def _basic_get_request(self, path, field): """ Perform a GET request and get the contents of the JSON response. @@ -94,10 +131,9 @@ def _basic_get_request(self, path, field, raise_error=True): returns something like {"apps": [ {"app1"}, {"app2"} ]}. We're interested in the contents of "apps". """ - d = self.get_json(path) - return d.addCallback(self._get_json_field, field, raise_error) + return self.get_json(path).addCallback(self._get_json_field, field) - def _get_json_field(self, response_json, field_name, raise_error=True): + def _get_json_field(self, response_json, field_name): """ Get a JSON field from the response JSON. @@ -105,17 +141,11 @@ def _get_json_field(self, response_json, field_name, raise_error=True): The parsed JSON content of the response. :param: field_name: The name of the field in the JSON to get. - :param: raise_error: - If True, an error will be raised if the field is not present in - the response JSON. If False, None is returned. """ if field_name not in response_json: - if raise_error: - raise KeyError('Unable to get value for "%s" from Marathon ' - 'response: "%s"' % ( - field_name, str(response_json),)) - else: - return None + raise KeyError('Unable to get value for "%s" from Marathon ' + 'response: "%s"' % ( + field_name, json.dumps(response_json),)) return response_json[field_name] @@ -135,7 +165,7 @@ def post_event_subscription(self, callback_url): 'POST', '/v2/eventSubscriptions?%s' % urlencode({ 'callbackUrl': callback_url, })) - return d.addCallback(lambda response: response.code == 200) + return d.addCallback(lambda response: response.code == OK) def get_apps(self): """ @@ -150,13 +180,12 @@ def get_app(self, app_id): """ return self._basic_get_request('/v2/apps%s' % (app_id,), 'app') - def get_app_tasks(self, app_id, raise_error=True): + def get_app_tasks(self, app_id): """ Get the currently running tasks for the app with the given app ID, returning a list of task definitions. """ - return self._basic_get_request( - '/v2/apps%s/tasks' % (app_id,), 'tasks', raise_error) + return self._basic_get_request('/v2/apps%s/tasks' % (app_id,), 'tasks') class ConsulClient(JsonClient): @@ -198,7 +227,7 @@ def register_agent_service_fallback(self, failure, registration): log.msg('Falling back for %s at %s.' % ( registration['Name'], self.endpoint)) return self.request( - 'PUT', '/v1/agent/service/register', json_data=registration, + 'PUT', '/v1/agent/service/register', json_data=registration, timeout=self.fallback_timeout) def deregister_agent_service(self, agent_endpoint, service_id): @@ -229,27 +258,8 @@ def get_kv_keys(self, keys_path, separator=None): params = {'keys': ''} if separator: params['separator'] = separator - d = self.request('GET', '/v1/kv/%s?%s' % (quote(keys_path), - urlencode(params))) - return d.addCallback(self._get_kv_keys_from_response, keys_path) - - def _get_kv_keys_from_response(self, response, keys_path): - """ - Get the list of keys from a Consul k/v store response. If the keys were - not found (Consul returned a 404), return an empty list. - """ - if response.code == 200: - return response.json() - elif response.code == 404: - if self.debug: - log.msg( - 'Consul returned a 404 when getting the keys for %s' % ( - keys_path,)) - return [] - else: - raise RuntimeError('Unexpected response from Consul when getting ' - 'keys for "%s", status code = %s' % ( - keys_path, response.code,)) + return self.get_json( + '/v1/kv/%s?%s' % (quote(keys_path), urlencode(params),)) def delete_kv_keys(self, key, recurse=False): """ diff --git a/consular/main.py b/consular/main.py index eadf356..7d9726a 100644 --- a/consular/main.py +++ b/consular/main.py @@ -1,11 +1,13 @@ import json -from consular.clients import ConsulClient, MarathonClient +from consular.clients import ( + ConsulClient, MarathonClient, UnexpectedResponseError) from twisted.internet import reactor from twisted.web import server from twisted.internet.defer import ( succeed, inlineCallbacks, returnValue, gatherResults) +from twisted.web.http import NOT_FOUND from twisted.python import log from klein import Klein @@ -389,6 +391,7 @@ def clean_consul_app_labels(self, app_name, labels): """ # Get the existing labels from Consul d = self.get_consul_app_keys(app_name) + d.addErrback(self._ignore_not_found_error, []) # Filter out the Marathon labels d.addCallback(self._filter_marathon_labels, labels) @@ -454,6 +457,7 @@ def purge_dead_app_labels(self, apps): """ # Get the existing keys d = self.get_consul_consular_keys() + d.addErrback(self._ignore_not_found_error, []) # Filter the present apps out d.addCallback(self._filter_marathon_apps, apps) @@ -461,6 +465,22 @@ def purge_dead_app_labels(self, apps): # Delete the remaining keys return d.addCallback(self.delete_consul_kv_keys, recurse=True) + def _ignore_not_found_error(self, failure, return_value=None): + """ + Handle `UnexpectedResponseError`s from requests by ignoring not found + (404) responses and returning the given return value. Other errors + will be re-raised. + """ + # Re-raise any unknown error types + failure.trap(UnexpectedResponseError) + + # If not found, return the return value + if failure.value.response.code == NOT_FOUND: + return return_value + + # Don't know what the response is, re-raise the exception + failure.raiseException() + def _filter_marathon_apps(self, consul_keys, marathon_apps): """ Takes a list of Consul keys and removes those with keys not found in @@ -517,7 +537,8 @@ def purge_dead_agent_services(self, agent_endpoint): def purge_service_if_dead(self, agent_endpoint, app_id, consul_task_ids): # Get the running tasks for the app (don't raise an error if the tasks # are not found) - d = self.marathon_client.get_app_tasks(app_id, raise_error=False) + d = self.marathon_client.get_app_tasks(app_id) + d.addErrback(self._ignore_not_found_error, []) # Remove the running tasks from the set of Consul services d.addCallback(self._filter_marathon_tasks, consul_task_ids) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 7c1c83e..6285fe3 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -9,6 +9,7 @@ from twisted.web.client import HTTPConnectionPool from twisted.python import log +from consular.clients import UnexpectedResponseError from consular.main import Consular import treq @@ -320,13 +321,13 @@ def test_register_with_marathon_unexpected_response(self): 'instance. Please re-start this instance with ' '"--event_subscriber http_callback".'}))) - failure = self.failureResultOf(d, KeyError) + failure = self.failureResultOf(d, UnexpectedResponseError) self.assertEqual( failure.getErrorMessage(), - '\'Unable to get value for "callbackUrls" from Marathon response: ' - '"{u\\\'message\\\': u\\\'http event callback system is not ' - 'running on this Marathon instance. Please re-start this instance ' - 'with "--event_subscriber http_callback".\\\'}"\'') + 'response: code=400, body={"message": "http event callback system ' + 'is not running on this Marathon instance. Please re-start this ' + 'instance with \\"--event_subscriber http_callback\\"."} ' + '\nrequest: method=, url=, body=') @inlineCallbacks def test_sync_app_task(self): @@ -508,11 +509,10 @@ def test_sync_app_labels_cleanup_forbidden(self): # Error is raised into a DeferredList, must get actual error failure = self.failureResultOf(d, FirstError) actual_failure = failure.value.subFailure - self.assertEqual(actual_failure.type, RuntimeError) + self.assertEqual(actual_failure.type, UnexpectedResponseError) self.assertEqual( actual_failure.getErrorMessage(), - 'Unexpected response from Consul when getting keys for ' - '"consular/my-app", status code = 403') + 'response: code=403, body=None \nrequest: method=, url=, body=') @inlineCallbacks def test_sync_app(self): @@ -555,6 +555,29 @@ def test_sync_apps(self): FakeResponse(200, [], json.dumps({'apps': []}))) yield d + @inlineCallbacks + def test_sync_apps_field_not_found(self): + """ + When syncing apps, and Marathon returns a JSON response with an + unexpected structure (the "apps" field is missing. A KeyError should + be raised. + """ + d = self.consular.sync_apps(purge=False) + marathon_request = yield self.requests.get() + self.assertEqual(marathon_request['url'], + 'http://localhost:8080/v2/apps') + self.assertEqual(marathon_request['method'], 'GET') + marathon_request['deferred'].callback( + FakeResponse(200, [], json.dumps({ + 'some field': 'that was unexpected' + }))) + + failure = self.failureResultOf(d, KeyError) + self.assertEqual( + failure.getErrorMessage(), + '\'Unable to get value for "apps" from Marathon response: "{"some ' + 'field": "that was unexpected"}"\'') + def test_check_apps_namespace_clash_no_clash(self): """ When checking for app namespace clashes and there are no clashes, the @@ -731,8 +754,85 @@ def test_purge_old_services(self): 'http://localhost:8080/v2/apps/testingapp/tasks') self.assertEqual(testingapp_request['method'], 'GET') testingapp_request['deferred'].callback( - FakeResponse(200, [], json.dumps({})) + FakeResponse(200, [], json.dumps({'tasks': []}))) + + # Expecting a service deregistering in Consul as a result. Only the + # task with the correct tag is returned. + deregister_request = yield self.requests.get() + self.assertEqual( + deregister_request['url'], + ('http://1.2.3.4:8500/v1/agent/service/deregister/' + 'testingapp.someid1')) + self.assertEqual(deregister_request['method'], 'PUT') + deregister_request['deferred'].callback( + FakeResponse(200, [], json.dumps({}))) + yield d + + @inlineCallbacks + def test_purge_old_services_tasks_not_found(self): + """ + Services previously registered with Consul by Consular but that no + longer exist in Marathon should be purged if a registration ID is set, + even if the tasks are not found. + """ + d = self.consular.purge_dead_services() + consul_request = yield self.requests.get() + self.assertEqual( + consul_request['url'], + 'http://localhost:8500/v1/catalog/nodes') + consul_request['deferred'].callback( + FakeResponse(200, [], json.dumps([{ + 'Node': 'consul-node', + 'Address': '1.2.3.4', + }])) ) + agent_request = yield self.requests.get() + # Expecting a request to list of all services in Consul, returning 3 + # services - one tagged with our registration ID, one tagged with a + # different registration ID, and one with no tags. + self.assertEqual( + agent_request['url'], + 'http://1.2.3.4:8500/v1/agent/services') + self.assertEqual(agent_request['method'], 'GET') + agent_request['deferred'].callback( + FakeResponse(200, [], json.dumps({ + "testingapp.someid1": { + "ID": "testingapp.someid1", + "Service": "testingapp", + "Tags": [ + "consular-reg-id=test", + "consular-app-id=/testingapp", + ], + "Address": "machine-1", + "Port": 8102 + }, + "testingapp.someid2": { + "ID": "testingapp.someid2", + "Service": "testingapp", + "Tags": [ + "consular-reg-id=blah", + "consular-app-id=/testingapp", + ], + "Address": "machine-2", + "Port": 8103 + }, + "testingapp.someid3": { + "ID": "testingapp.someid2", + "Service": "testingapp", + "Tags": None, + "Address": "machine-2", + "Port": 8104 + } + })) + ) + + # Expecting a request for the tasks for a given app, returning a 404 + testingapp_request = yield self.requests.get() + self.assertEqual(testingapp_request['url'], + 'http://localhost:8080/v2/apps/testingapp/tasks') + self.assertEqual(testingapp_request['method'], 'GET') + testingapp_request['deferred'].callback( + FakeResponse(404, [], None)) # Expecting a service deregistering in Consul as a result. Only the # task with the correct tag is returned. @@ -856,11 +956,10 @@ def test_purge_dead_app_labels_forbidden(self): # Return a 403 error consul_request['deferred'].callback(FakeResponse(403, [], None)) - failure = self.failureResultOf(d, RuntimeError) + failure = self.failureResultOf(d, UnexpectedResponseError) self.assertEqual( failure.getErrorMessage(), - 'Unexpected response from Consul when getting keys for ' - '"consular/", status code = 403') + 'response: code=403, body=None \nrequest: method=, url=, body=') @inlineCallbacks def test_fallback_to_main_consul(self): From a8726154ce8465de008714ac10dce783f20def94 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Fri, 18 Sep 2015 15:04:58 +0200 Subject: [PATCH 043/111] Tweak some comments --- consular/clients.py | 3 ++- consular/tests/test_main.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/consular/clients.py b/consular/clients.py index e494816..f841047 100644 --- a/consular/clients.py +++ b/consular/clients.py @@ -37,7 +37,8 @@ def _log_http_error(self, failure, url): def request(self, method, path, endpoint=None, json_data=None, **kwargs): """ Perform a request. A number of basic defaults are set on the request - that make using a JSON API easier. + that make using a JSON API easier. These defaults can be overridden by + setting the parameters in the keyword args. :param: method: The HTTP method to use (example is `GET`). diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 6285fe3..2dcc0c5 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -559,8 +559,8 @@ def test_sync_apps(self): def test_sync_apps_field_not_found(self): """ When syncing apps, and Marathon returns a JSON response with an - unexpected structure (the "apps" field is missing. A KeyError should - be raised. + unexpected structure (the "apps" field is missing). A KeyError should + be raised with a useful message. """ d = self.consular.sync_apps(purge=False) marathon_request = yield self.requests.get() From 8a28be143f8f4cb1ed05317e26823f2875968f93 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Mon, 19 Oct 2015 09:05:39 +0200 Subject: [PATCH 044/111] Oh golang... --- templates/nginx.ctmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/nginx.ctmpl b/templates/nginx.ctmpl index 078b52a..7132da9 100644 --- a/templates/nginx.ctmpl +++ b/templates/nginx.ctmpl @@ -7,7 +7,7 @@ # Nginx. If the key does not exist the service isn't added to the # list of services in the Nginx config. -{{range services}}{{$labels = ls (print "consular/" .Name) | explode }}{{if $labels.domain}} +{{range services}}{{$labels := ls (print "consular/" .Name) | explode }}{{if $labels.domain}} upstream {{.Name}} { {{range service .Name }}server {{.Address}}:{{.Port}}; From 21fa6408f2a2e5a9f855140be6b52f6a57b335c3 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Mon, 19 Oct 2015 15:59:08 +0200 Subject: [PATCH 045/111] Replace all gatherResults calls with inlineCallbacks/yield * Serialize everything the easiest way possible for now --- consular/main.py | 64 +++++++++++++++++++------------------ consular/tests/test_main.py | 7 ++-- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/consular/main.py b/consular/main.py index 7d9726a..290dc2f 100644 --- a/consular/main.py +++ b/consular/main.py @@ -5,8 +5,7 @@ from twisted.internet import reactor from twisted.web import server -from twisted.internet.defer import ( - succeed, inlineCallbacks, returnValue, gatherResults) +from twisted.internet.defer import succeed, inlineCallbacks, returnValue from twisted.web.http import NOT_FOUND from twisted.python import log @@ -306,11 +305,13 @@ def sync_apps(self, purge=False): d.addCallback(self.check_apps_namespace_clash) return d.addCallback(self.sync_and_purge_apps, purge) + @inlineCallbacks def sync_and_purge_apps(self, apps, purge=False): - deferreds = [gatherResults([self.sync_app(app) for app in apps])] + for app in apps: + yield self.sync_app(app) + if purge: - deferreds.append(self.purge_dead_apps(apps)) - return gatherResults(deferreds) + yield self.purge_dead_apps(apps) def check_apps_namespace_clash(self, apps): """ @@ -341,18 +342,17 @@ def check_apps_namespace_clash(self, apps): return apps + @inlineCallbacks def sync_app(self, app): - return gatherResults([ - self.sync_app_labels(app), - self.sync_app_tasks(app), - ]) + yield self.sync_app_labels(app) + yield self.sync_app_tasks(app) + @inlineCallbacks def purge_dead_apps(self, apps): - return gatherResults([ - self.purge_dead_services(), - self.purge_dead_app_labels(apps) - ]) + yield self.purge_dead_services() + yield self.purge_dead_app_labels(apps) + @inlineCallbacks def sync_app_labels(self, app): """ Sync the app labels for the given app by pushing its labels to the @@ -366,10 +366,8 @@ def sync_app_labels(self, app): # we're already connected to, they're not local to the agents. app_name = get_app_name(app['id']) labels = app.get('labels', {}) - return gatherResults([ - self.put_consul_app_labels(app_name, labels), - self.clean_consul_app_labels(app_name, labels) - ]) + yield self.put_consul_app_labels(app_name, labels) + yield self.clean_consul_app_labels(app_name, labels) def put_consul_app_labels(self, app_name, labels): """ @@ -379,10 +377,11 @@ def put_consul_app_labels(self, app_name, labels): return self.put_consul_kvs({'consular/%s/%s' % (app_name, key,): value for key, value in labels.items()}) + @inlineCallbacks def put_consul_kvs(self, key_values): """ Store the given key/value set in the Consul k/v store. """ - return gatherResults([self.consul_client.put_kv(key, value) - for key, value in key_values.items()]) + for key, value in key_values.items(): + yield self.consul_client.put_kv(key, value) def clean_consul_app_labels(self, app_name, labels): """ @@ -410,10 +409,11 @@ def get_consul_consular_keys(self): """ return self.consul_client.get_kv_keys('consular/', separator='/') + @inlineCallbacks def delete_consul_kv_keys(self, keys, recurse=False): """ Delete a sequence of Consul k/v keys. """ - return gatherResults([self.consul_client.delete_kv_keys(key, recurse) - for key in keys]) + for key in keys: + yield self.consul_client.delete_kv_keys(key, recurse) def _filter_marathon_labels(self, consul_keys, marathon_labels): """ @@ -437,10 +437,11 @@ def _consul_key_to_marathon_label_key(self, consul_key): """ return consul_key.split('/', 2)[-1] + @inlineCallbacks def sync_app_tasks(self, app): - d = self.marathon_client.get_app_tasks(app['id']) - return d.addCallback(lambda tasks: gatherResults( - self.sync_app_task(app, task) for task in tasks)) + tasks = yield self.marathon_client.get_app_tasks(app['id']) + for task in tasks: + yield self.sync_app_task(app, task) def sync_app_task(self, app, task): return self.register_service( @@ -503,12 +504,12 @@ def _consul_key_to_marathon_app_name(self, consul_key): """ return consul_key.split('/', 1)[-1] + @inlineCallbacks def purge_dead_services(self): - d = self.consul_client.get_catalog_nodes() - return d.addCallback(lambda data: gatherResults([ + nodes = yield self.consul_client.get_catalog_nodes() + for node in nodes: self.purge_dead_agent_services( - get_agent_endpoint(node['Address'])) for node in data - ])) + get_agent_endpoint(node['Address'])) @inlineCallbacks def purge_dead_agent_services(self, agent_endpoint): @@ -534,6 +535,7 @@ def purge_dead_agent_services(self, agent_endpoint): for app_id, task_ids in services.items(): yield self.purge_service_if_dead(agent_endpoint, app_id, task_ids) + @inlineCallbacks def purge_service_if_dead(self, agent_endpoint, app_id, consul_task_ids): # Get the running tasks for the app (don't raise an error if the tasks # are not found) @@ -544,9 +546,9 @@ def purge_service_if_dead(self, agent_endpoint, app_id, consul_task_ids): d.addCallback(self._filter_marathon_tasks, consul_task_ids) # Deregister the remaining old services - return d.addCallback(lambda service_ids: gatherResults( - [self.deregister_service(agent_endpoint, app_id, service_id) - for service_id in service_ids])) + service_ids = yield d + for service_id in service_ids: + yield self.deregister_service(agent_endpoint, app_id, service_id) def _filter_marathon_tasks(self, marathon_tasks, consul_service_ids): if not marathon_tasks: diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 2dcc0c5..38a274d 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -507,11 +507,10 @@ def test_sync_app_labels_cleanup_forbidden(self): get_request['deferred'].callback(FakeResponse(403, [], None)) # Error is raised into a DeferredList, must get actual error - failure = self.failureResultOf(d, FirstError) - actual_failure = failure.value.subFailure - self.assertEqual(actual_failure.type, UnexpectedResponseError) + failure = self.failureResultOf(d, UnexpectedResponseError) + self.assertEqual(failure.type, UnexpectedResponseError) self.assertEqual( - actual_failure.getErrorMessage(), + failure.getErrorMessage(), 'response: code=403, body=None \nrequest: method=, url=, body=') @inlineCallbacks From 0e19a0126aa42275ac244a94cb048e321073a273 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Mon, 19 Oct 2015 16:02:58 +0200 Subject: [PATCH 046/111] Remove unused import --- consular/tests/test_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 38a274d..a2b79fc 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -5,7 +5,7 @@ from twisted.web.server import Site from twisted.internet import reactor from twisted.internet.defer import ( - inlineCallbacks, DeferredQueue, Deferred, FirstError, succeed) + inlineCallbacks, DeferredQueue, Deferred, succeed) from twisted.web.client import HTTPConnectionPool from twisted.python import log From f9d51c2dfeb25cfcdf17351c52067076c1553a0a Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Mon, 19 Oct 2015 16:23:15 +0200 Subject: [PATCH 047/111] Fix misnamed attribute (@miltontony) --- consular/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consular/main.py b/consular/main.py index 290dc2f..d08cbe1 100644 --- a/consular/main.py +++ b/consular/main.py @@ -29,7 +29,7 @@ class ConsularSite(server.Site): debug = False def log(self, request): - if self._debug: + if self.debug: server.Site.log(self, request) From 88b27c566888bb52eb3214c6d38bedd6bba0c18b Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Mon, 19 Oct 2015 16:51:03 +0200 Subject: [PATCH 048/111] Remove the fancy errback to handle 404s, use inlineCallbacks --- consular/main.py | 56 ++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/consular/main.py b/consular/main.py index d08cbe1..62747b3 100644 --- a/consular/main.py +++ b/consular/main.py @@ -383,20 +383,26 @@ def put_consul_kvs(self, key_values): for key, value in key_values.items(): yield self.consul_client.put_kv(key, value) + @inlineCallbacks def clean_consul_app_labels(self, app_name, labels): """ Delete app labels stored in the Consul k/v store under the given app name that aren't present in the given set of labels. """ # Get the existing labels from Consul - d = self.get_consul_app_keys(app_name) - d.addErrback(self._ignore_not_found_error, []) + try: + keys = yield self.get_consul_app_keys(app_name) + except UnexpectedResponseError as e: + if e.response.code == NOT_FOUND: + keys = [] + else: + raise e # Filter out the Marathon labels - d.addCallback(self._filter_marathon_labels, labels) + keys = self._filter_marathon_labels(keys, labels) # Delete the non-existant keys - return d.addCallback(self.delete_consul_kv_keys) + yield self.delete_consul_kv_keys(keys) def get_consul_app_keys(self, app_name): """ Get the Consul k/v keys for the app with the given name. """ @@ -448,6 +454,7 @@ def sync_app_task(self, app, task): get_agent_endpoint(task['host']), app['id'], task['id'], task['host'], task['ports'][0]) + @inlineCallbacks def purge_dead_app_labels(self, apps): """ Delete any keys stored in the Consul k/v store that belong to apps that @@ -457,30 +464,19 @@ def purge_dead_app_labels(self, apps): The list of apps as returned by the Marathon API. """ # Get the existing keys - d = self.get_consul_consular_keys() - d.addErrback(self._ignore_not_found_error, []) + try: + keys = yield self.get_consul_consular_keys() + except UnexpectedResponseError as e: + if e.response.code == NOT_FOUND: + keys = [] + else: + raise e # Filter the present apps out - d.addCallback(self._filter_marathon_apps, apps) + keys = self._filter_marathon_apps(keys, apps) # Delete the remaining keys - return d.addCallback(self.delete_consul_kv_keys, recurse=True) - - def _ignore_not_found_error(self, failure, return_value=None): - """ - Handle `UnexpectedResponseError`s from requests by ignoring not found - (404) responses and returning the given return value. Other errors - will be re-raised. - """ - # Re-raise any unknown error types - failure.trap(UnexpectedResponseError) - - # If not found, return the return value - if failure.value.response.code == NOT_FOUND: - return return_value - - # Don't know what the response is, re-raise the exception - failure.raiseException() + yield self.delete_consul_kv_keys(keys, recurse=True) def _filter_marathon_apps(self, consul_keys, marathon_apps): """ @@ -539,14 +535,18 @@ def purge_dead_agent_services(self, agent_endpoint): def purge_service_if_dead(self, agent_endpoint, app_id, consul_task_ids): # Get the running tasks for the app (don't raise an error if the tasks # are not found) - d = self.marathon_client.get_app_tasks(app_id) - d.addErrback(self._ignore_not_found_error, []) + try: + tasks = yield self.marathon_client.get_app_tasks(app_id) + except UnexpectedResponseError as e: + if e.response.code == NOT_FOUND: + tasks = [] + else: + raise e # Remove the running tasks from the set of Consul services - d.addCallback(self._filter_marathon_tasks, consul_task_ids) + service_ids = self._filter_marathon_tasks(tasks, consul_task_ids) # Deregister the remaining old services - service_ids = yield d for service_id in service_ids: yield self.deregister_service(agent_endpoint, app_id, service_id) From ef66d6aaeb94f0da6d39b4b82b40eed9d7e35da9 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Mon, 19 Oct 2015 16:55:59 +0200 Subject: [PATCH 049/111] Add a message when deleting Consul k/v's --- consular/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/consular/main.py b/consular/main.py index 62747b3..9e86f87 100644 --- a/consular/main.py +++ b/consular/main.py @@ -419,6 +419,8 @@ def get_consul_consular_keys(self): def delete_consul_kv_keys(self, keys, recurse=False): """ Delete a sequence of Consul k/v keys. """ for key in keys: + log.msg('Deleting Consul k/v key "%s", recursively? %s' % ( + key, recurse)) yield self.consul_client.delete_kv_keys(key, recurse) def _filter_marathon_labels(self, consul_keys, marathon_labels): From 0cfe4fac1e6bbaa81a8d907f177b8a06146bac34 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Mon, 19 Oct 2015 17:11:04 +0200 Subject: [PATCH 050/111] Add some more log messages when deleting Consul k/v keys --- consular/main.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/consular/main.py b/consular/main.py index 9e86f87..8efb8bf 100644 --- a/consular/main.py +++ b/consular/main.py @@ -390,6 +390,7 @@ def clean_consul_app_labels(self, app_name, labels): name that aren't present in the given set of labels. """ # Get the existing labels from Consul + log.msg('Cleaning labels no longer in use by app "%s"' % app_name) try: keys = yield self.get_consul_app_keys(app_name) except UnexpectedResponseError as e: @@ -398,9 +399,15 @@ def clean_consul_app_labels(self, app_name, labels): else: raise e + log.msg('%d labels stored in Marathon, %d keys found in Consul for app' + ' "%s"' % (len(labels), len(keys), app_name)) + # Filter out the Marathon labels keys = self._filter_marathon_labels(keys, labels) + log.msg('%d keys to be deleted from Consul for app %s' % ( + len(keys), app_name)) + # Delete the non-existant keys yield self.delete_consul_kv_keys(keys) @@ -465,6 +472,7 @@ def purge_dead_app_labels(self, apps): :param: apps: The list of apps as returned by the Marathon API. """ + log.msg('Purging dead app labels') # Get the existing keys try: keys = yield self.get_consul_consular_keys() @@ -474,8 +482,12 @@ def purge_dead_app_labels(self, apps): else: raise e + log.msg('Got %d keys from Consul' % len(keys)) + # Filter the present apps out keys = self._filter_marathon_apps(keys, apps) + log.msg('After filtering out running apps, %d Consul keys remain to be' + 'purged' % len(keys)) # Delete the remaining keys yield self.delete_consul_kv_keys(keys, recurse=True) From 1689492fdcb41d7ac31c7a681c543bb02916a62a Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Mon, 19 Oct 2015 17:23:56 +0200 Subject: [PATCH 051/111] Disable broken app label purging (@miltontony) --- consular/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consular/main.py b/consular/main.py index 8efb8bf..fe22379 100644 --- a/consular/main.py +++ b/consular/main.py @@ -350,7 +350,7 @@ def sync_app(self, app): @inlineCallbacks def purge_dead_apps(self, apps): yield self.purge_dead_services() - yield self.purge_dead_app_labels(apps) + # yield self.purge_dead_app_labels(apps) @inlineCallbacks def sync_app_labels(self, app): From 32104a9ff2e43d6e92df8772672a396c919bcff1 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 20 Oct 2015 10:20:02 +0200 Subject: [PATCH 052/111] Handle 404 not found errors in one place * Add logging for when things are not found --- consular/main.py | 56 ++++++++++++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/consular/main.py b/consular/main.py index fe22379..3daa024 100644 --- a/consular/main.py +++ b/consular/main.py @@ -24,6 +24,28 @@ def get_agent_endpoint(host): return 'http://%s:8500' % (host,) +@inlineCallbacks +def handle_not_found_error(f, *args, **kwargs): + """ + Perform a request and catch the not found (404) error if one occurs. + + :param: f: The function to call to perform the request. The function may + return a deferred. + :param: args: The arguments to call the function with. + :param: kwargs: The keyword arguments to call the function with. + :returns: The return value of the function call or None if there was a 404 + response code. + """ + try: + response = yield f(*args, **kwargs) + except UnexpectedResponseError as e: + if e.response.code == NOT_FOUND: + response = None + else: + raise e + returnValue(response) + + class ConsularSite(server.Site): debug = False @@ -391,13 +413,10 @@ def clean_consul_app_labels(self, app_name, labels): """ # Get the existing labels from Consul log.msg('Cleaning labels no longer in use by app "%s"' % app_name) - try: - keys = yield self.get_consul_app_keys(app_name) - except UnexpectedResponseError as e: - if e.response.code == NOT_FOUND: - keys = [] - else: - raise e + keys = yield handle_not_found_error(self.get_consul_app_keys, app_name) + if keys is None: + log.msg('No keys found in Consul for service "%s"' % app_name) + return log.msg('%d labels stored in Marathon, %d keys found in Consul for app' ' "%s"' % (len(labels), len(keys), app_name)) @@ -474,13 +493,10 @@ def purge_dead_app_labels(self, apps): """ log.msg('Purging dead app labels') # Get the existing keys - try: - keys = yield self.get_consul_consular_keys() - except UnexpectedResponseError as e: - if e.response.code == NOT_FOUND: - keys = [] - else: - raise e + keys = yield handle_not_found_error(self.get_consul_consular_keys) + if keys is None: + log.msg('No Consular keys found in Consul') + return log.msg('Got %d keys from Consul' % len(keys)) @@ -549,13 +565,11 @@ def purge_dead_agent_services(self, agent_endpoint): def purge_service_if_dead(self, agent_endpoint, app_id, consul_task_ids): # Get the running tasks for the app (don't raise an error if the tasks # are not found) - try: - tasks = yield self.marathon_client.get_app_tasks(app_id) - except UnexpectedResponseError as e: - if e.response.code == NOT_FOUND: - tasks = [] - else: - raise e + tasks = yield handle_not_found_error( + self.marathon_client.get_app_tasks, app_id) + if tasks is None: + log.msg('No tasks found in Marathon for app ID "%s"' % app_id) + tasks = [] # Remove the running tasks from the set of Consul services service_ids = self._filter_marathon_tasks(tasks, consul_task_ids) From 0204c9dfa327c1835de294ae1cdf78eb027436b0 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 20 Oct 2015 10:29:10 +0200 Subject: [PATCH 053/111] Remove method to delete list of keys --- consular/main.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/consular/main.py b/consular/main.py index 3daa024..ae89a94 100644 --- a/consular/main.py +++ b/consular/main.py @@ -428,7 +428,8 @@ def clean_consul_app_labels(self, app_name, labels): len(keys), app_name)) # Delete the non-existant keys - yield self.delete_consul_kv_keys(keys) + for key in keys: + yield self.consul_client.delete_kv_keys(key) def get_consul_app_keys(self, app_name): """ Get the Consul k/v keys for the app with the given name. """ @@ -441,14 +442,6 @@ def get_consul_consular_keys(self): """ return self.consul_client.get_kv_keys('consular/', separator='/') - @inlineCallbacks - def delete_consul_kv_keys(self, keys, recurse=False): - """ Delete a sequence of Consul k/v keys. """ - for key in keys: - log.msg('Deleting Consul k/v key "%s", recursively? %s' % ( - key, recurse)) - yield self.consul_client.delete_kv_keys(key, recurse) - def _filter_marathon_labels(self, consul_keys, marathon_labels): """ Takes a list of Consul keys and removes those with keys not found in @@ -506,7 +499,8 @@ def purge_dead_app_labels(self, apps): 'purged' % len(keys)) # Delete the remaining keys - yield self.delete_consul_kv_keys(keys, recurse=True) + for key in keys: + yield self.consul_client.delete_kv_keys(key, recurse=True) def _filter_marathon_apps(self, consul_keys, marathon_apps): """ From 6d953d1d66631a406755f63126fa7db09eab81bb Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 20 Oct 2015 10:57:45 +0200 Subject: [PATCH 054/111] Wrap most of the new log messages in debug conditionals --- consular/main.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/consular/main.py b/consular/main.py index ae89a94..255c4a7 100644 --- a/consular/main.py +++ b/consular/main.py @@ -412,20 +412,24 @@ def clean_consul_app_labels(self, app_name, labels): name that aren't present in the given set of labels. """ # Get the existing labels from Consul - log.msg('Cleaning labels no longer in use by app "%s"' % app_name) + if self._debug: + log.msg('Cleaning labels no longer in use by app "%s"' % app_name) + keys = yield handle_not_found_error(self.get_consul_app_keys, app_name) if keys is None: log.msg('No keys found in Consul for service "%s"' % app_name) return - log.msg('%d labels stored in Marathon, %d keys found in Consul for app' - ' "%s"' % (len(labels), len(keys), app_name)) + if self._debug: + log.msg('%d labels stored in Marathon, %d keys found in Consul ' + 'for app "%s"' % (len(labels), len(keys), app_name)) # Filter out the Marathon labels keys = self._filter_marathon_labels(keys, labels) - log.msg('%d keys to be deleted from Consul for app %s' % ( - len(keys), app_name)) + if self._debug: + log.msg('%d keys to be deleted from Consul for app %s' % ( + len(keys), app_name)) # Delete the non-existant keys for key in keys: @@ -491,12 +495,15 @@ def purge_dead_app_labels(self, apps): log.msg('No Consular keys found in Consul') return - log.msg('Got %d keys from Consul' % len(keys)) + if self._debug: + log.msg('Got %d keys from Consul' % len(keys)) # Filter the present apps out keys = self._filter_marathon_apps(keys, apps) - log.msg('After filtering out running apps, %d Consul keys remain to be' - 'purged' % len(keys)) + + if self._debug: + log.msg('After filtering out running apps, %d Consul keys remain ' + 'to be purged' % len(keys)) # Delete the remaining keys for key in keys: From 159305d30b6b16aa3d394c7fe6f48bec0a760f9d Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 20 Oct 2015 11:08:00 +0200 Subject: [PATCH 055/111] Clean up sync apps method * Remove intermediary sync_apps_and_purge method * With inlineCallbacks it's functionally equivalent * Add a log message --- consular/main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/consular/main.py b/consular/main.py index 255c4a7..9117880 100644 --- a/consular/main.py +++ b/consular/main.py @@ -311,6 +311,7 @@ def deregister_service(self, agent_endpoint, app_id, service_id): return self.consul_client.deregister_agent_service( agent_endpoint, service_id) + @inlineCallbacks def sync_apps(self, purge=False): """ Ensure all the apps in Marathon are registered as services @@ -323,12 +324,11 @@ def sync_apps(self, purge=False): :param bool purge: To purge or not to purge. """ - d = self.marathon_client.get_apps() - d.addCallback(self.check_apps_namespace_clash) - return d.addCallback(self.sync_and_purge_apps, purge) + log.msg('Syncing apps') + apps = yield self.marathon_client.get_apps() + + self.check_apps_namespace_clash(apps) - @inlineCallbacks - def sync_and_purge_apps(self, apps, purge=False): for app in apps: yield self.sync_app(app) From 8b24241fb5f3785a89687e0b2f67b0ea4a1f9486 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 20 Oct 2015 11:11:46 +0200 Subject: [PATCH 056/111] Add a TODO note for app label purging --- consular/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/consular/main.py b/consular/main.py index 9117880..9c16e09 100644 --- a/consular/main.py +++ b/consular/main.py @@ -372,6 +372,7 @@ def sync_app(self, app): @inlineCallbacks def purge_dead_apps(self, apps): yield self.purge_dead_services() + # TODO: Fix app label purging (see issue #43) # yield self.purge_dead_app_labels(apps) @inlineCallbacks From 315cabe848fbf4df44ae38b3d37aaaa34206eaaf Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 20 Oct 2015 11:17:06 +0200 Subject: [PATCH 057/111] Add an extra log message when purging apps --- consular/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/consular/main.py b/consular/main.py index 9c16e09..ba03d44 100644 --- a/consular/main.py +++ b/consular/main.py @@ -333,6 +333,7 @@ def sync_apps(self, purge=False): yield self.sync_app(app) if purge: + log.msg('Purging dead apps') yield self.purge_dead_apps(apps) def check_apps_namespace_clash(self, apps): From 98aab7a0f1bf1edd32d3d7fb8f7712d09a135707 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 20 Oct 2015 12:14:58 +0200 Subject: [PATCH 058/111] Handle 2xx response codes as successes * Copy code from requests (as a callback) * Raise exception for all calls (not just get_json()) --- consular/clients.py | 53 +++++++++++++++++-------------------- consular/main.py | 5 ++-- consular/tests/test_main.py | 21 +++++++-------- 3 files changed, 36 insertions(+), 43 deletions(-) diff --git a/consular/clients.py b/consular/clients.py index f841047..7cbf17e 100644 --- a/consular/clients.py +++ b/consular/clients.py @@ -75,50 +75,45 @@ def request(self, method, path, endpoint=None, json_data=None, **kwargs): if self.debug: d.addCallback(self._log_http_response, method, url, data) - return d.addErrback(self._log_http_error, url) + d.addErrback(self._log_http_error, url) + return d.addCallback(self._raise_for_status, url) def get_json(self, path, **kwargs): """ Perform a GET request to the given path and return the JSON response. """ d = self.request('GET', path, **kwargs) - return d.addCallback(self._response_json_if_ok) + return d.addCallback(lambda response: response.json()) - def _response_json_if_ok(self, response): + def _raise_for_status(self, response, url): """ - Get the response JSON content if the respones code is OK (200), else - raise an `UnexpectedResponseError`. + Raises an `HTTPError` if the response did not succeed. + Adapted from the Requests library: + https://github.com/kennethreitz/requests/blob/v2.8.1/requests/models.py#L825-L837 """ - if response.code == OK: - return response.json() - else: - d = response.content() - d.addCallback(self._raise_unexpected_response_error, response) - return d + http_error_msg = '' - def _raise_unexpected_response_error(self, response_content, response): - raise UnexpectedResponseError(response, response_content) + if 400 <= response.code < 500: + http_error_msg = '%s Client Error for url: %s' % (response.code, + url) + elif 500 <= response.code < 600: + http_error_msg = '%s Server Error for url: %s' % (response.code, + url) -class UnexpectedResponseError(Exception): - """ - Error raised for a non-200 response code. - """ - def __init__(self, response, response_content, message=None): - if not message: - message = self._default_error_message(response, response_content) + if http_error_msg: + raise HTTPError(http_error_msg, response) - super(UnexpectedResponseError, self).__init__(message) - self.response = response + return response - def _default_error_message(self, response, response_content): - # Due to current testing method we can't get the Twisted Request object - # from the response and add extra useful fields to the error message. - # request = response.request - return 'response: code=%d, body=%s \nrequest: method=, url=, body=' % ( - response.code, response_content,) - # request.method, request.url, request.data)) +class HTTPError(IOError): + """ + Error raised for 4xx and 5xx response codes. + """ + def __init__(self, message, response): + self.response = response + super(HTTPError, self).__init__(message) class MarathonClient(JsonClient): diff --git a/consular/main.py b/consular/main.py index ba03d44..b7c303c 100644 --- a/consular/main.py +++ b/consular/main.py @@ -1,7 +1,6 @@ import json -from consular.clients import ( - ConsulClient, MarathonClient, UnexpectedResponseError) +from consular.clients import ConsulClient, MarathonClient, HTTPError from twisted.internet import reactor from twisted.web import server @@ -38,7 +37,7 @@ def handle_not_found_error(f, *args, **kwargs): """ try: response = yield f(*args, **kwargs) - except UnexpectedResponseError as e: + except HTTPError as e: if e.response.code == NOT_FOUND: response = None else: diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index a2b79fc..a775a79 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -9,7 +9,7 @@ from twisted.web.client import HTTPConnectionPool from twisted.python import log -from consular.clients import UnexpectedResponseError +from consular.clients import HTTPError from consular.main import Consular import treq @@ -321,13 +321,11 @@ def test_register_with_marathon_unexpected_response(self): 'instance. Please re-start this instance with ' '"--event_subscriber http_callback".'}))) - failure = self.failureResultOf(d, UnexpectedResponseError) + failure = self.failureResultOf(d, HTTPError) self.assertEqual( failure.getErrorMessage(), - 'response: code=400, body={"message": "http event callback system ' - 'is not running on this Marathon instance. Please re-start this ' - 'instance with \\"--event_subscriber http_callback\\"."} ' - '\nrequest: method=, url=, body=') + '400 Client Error for url: ' + 'http://localhost:8080/v2/eventSubscriptions') @inlineCallbacks def test_sync_app_task(self): @@ -507,11 +505,11 @@ def test_sync_app_labels_cleanup_forbidden(self): get_request['deferred'].callback(FakeResponse(403, [], None)) # Error is raised into a DeferredList, must get actual error - failure = self.failureResultOf(d, UnexpectedResponseError) - self.assertEqual(failure.type, UnexpectedResponseError) + failure = self.failureResultOf(d, HTTPError) self.assertEqual( failure.getErrorMessage(), - 'response: code=403, body=None \nrequest: method=, url=, body=') + '403 Client Error for url: ' + 'http://localhost:8500/v1/kv/consular/my-app?keys=') @inlineCallbacks def test_sync_app(self): @@ -955,10 +953,11 @@ def test_purge_dead_app_labels_forbidden(self): # Return a 403 error consul_request['deferred'].callback(FakeResponse(403, [], None)) - failure = self.failureResultOf(d, UnexpectedResponseError) + failure = self.failureResultOf(d, HTTPError) self.assertEqual( failure.getErrorMessage(), - 'response: code=403, body=None \nrequest: method=, url=, body=') + '403 Client Error for url: ' + 'http://localhost:8500/v1/kv/consular/?keys=&separator=%2F') @inlineCallbacks def test_fallback_to_main_consul(self): From d5c9613176737be2eb3a86318815579dcb198e53 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 20 Oct 2015 13:15:25 +0200 Subject: [PATCH 059/111] Make the test fail --- consular/tests/test_main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index a775a79..ade6223 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -900,8 +900,8 @@ def test_purge_dead_app_labels(self): # Return one existing app and one non-existing app consul_request['deferred'].callback( FakeResponse(200, [], json.dumps([ - 'consular/my-app', - 'consular/my-app2', + 'consular/my-app/', + 'consular/my-app2/', ])) ) @@ -910,7 +910,7 @@ def test_purge_dead_app_labels(self): self.assertEqual(consul_request['method'], 'DELETE') self.assertEqual( consul_request['url'], - 'http://localhost:8500/v1/kv/consular/my-app2?recurse') + 'http://localhost:8500/v1/kv/consular/my-app2/?recurse') consul_request['deferred'].callback( FakeResponse(200, [], json.dumps({}))) From bb8e556c6ed729c90cc78ff3f6551f2ba418e43e Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 20 Oct 2015 13:42:24 +0200 Subject: [PATCH 060/111] Fix app purging * When fetching keys from Consul with the 'keys' and 'separator' params, Consul returns keys with a trailing '/'. --- consular/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consular/main.py b/consular/main.py index b7c303c..82ccacb 100644 --- a/consular/main.py +++ b/consular/main.py @@ -530,7 +530,7 @@ def _consul_key_to_marathon_app_name(self, consul_key): Trims the 'consular/' from the front of the key path to get the Marathon app name. """ - return consul_key.split('/', 1)[-1] + return consul_key.split('/', 1)[-1].strip('/') @inlineCallbacks def purge_dead_services(self): From ffba48da043912a981cbcde28fdc96a78e0ebc91 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 20 Oct 2015 13:54:05 +0200 Subject: [PATCH 061/111] Re-enable app purging * Tested on qa-vumi cluster --- consular/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/consular/main.py b/consular/main.py index 82ccacb..de779a6 100644 --- a/consular/main.py +++ b/consular/main.py @@ -372,8 +372,7 @@ def sync_app(self, app): @inlineCallbacks def purge_dead_apps(self, apps): yield self.purge_dead_services() - # TODO: Fix app label purging (see issue #43) - # yield self.purge_dead_app_labels(apps) + yield self.purge_dead_app_labels(apps) @inlineCallbacks def sync_app_labels(self, app): From bb7f245c7d6a4fadf6e817a529a3446579730b02 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Mon, 26 Oct 2015 11:31:39 +0200 Subject: [PATCH 062/111] Move the sync looping call out of the CLI --- consular/cli.py | 4 +--- consular/main.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/consular/cli.py b/consular/cli.py index f516cbc..07abebb 100644 --- a/consular/cli.py +++ b/consular/cli.py @@ -54,7 +54,6 @@ def main(scheme, host, port, sync_interval, purge, logfile, debug, timeout, fallback, fallback_timeout): # pragma: no cover from consular.main import Consular - from twisted.internet.task import LoopingCall from twisted.internet import reactor from twisted.python import log @@ -72,8 +71,7 @@ def main(scheme, host, port, consular.register_marathon_event_callback(events_url) if sync_interval > 0: - lc = LoopingCall(consular.sync_apps, purge) - lc.start(sync_interval, now=True) + consular.schedule_sync(sync_interval, purge) consular.run(host, port) reactor.run() diff --git a/consular/main.py b/consular/main.py index de779a6..c27b42a 100644 --- a/consular/main.py +++ b/consular/main.py @@ -5,6 +5,7 @@ from twisted.internet import reactor from twisted.web import server from twisted.internet.defer import succeed, inlineCallbacks, returnValue +from twisted.internet.task import LoopingCall from twisted.web.http import NOT_FOUND from twisted.python import log @@ -108,6 +109,19 @@ def run(self, host, port): site.debug = self._debug self.clock.listenTCP(port, site, interface=host) + def schedule_sync(self, interval, purge=False): + """ + Schedule a recurring sync of apps, starting after this method is + called. + + :param float interval: + The number of seconds between syncs. + :param bool purge: + Whether to purge old apps after each sync. + """ + lc = LoopingCall(self.sync_apps, purge) + lc.start(interval, now=True) + @inlineCallbacks def register_marathon_event_callback(self, events_url): """ From 1af4eb764eba2eec41fd3109a506bbfcbf482f7e Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Mon, 26 Oct 2015 12:05:05 +0200 Subject: [PATCH 063/111] Add a test for scheduled syncing --- consular/main.py | 6 +++++- consular/tests/test_main.py | 38 +++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/consular/main.py b/consular/main.py index c27b42a..bd1b26a 100644 --- a/consular/main.py +++ b/consular/main.py @@ -118,9 +118,13 @@ def schedule_sync(self, interval, purge=False): The number of seconds between syncs. :param bool purge: Whether to purge old apps after each sync. + :return: + A tuple of the LoopingCall object and the deferred created when it + was started. """ lc = LoopingCall(self.sync_apps, purge) - lc.start(interval, now=True) + lc.clock = self.clock + return (lc, lc.start(interval, now=True)) @inlineCallbacks def register_marathon_event_callback(self, events_url): diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index ade6223..b2a7f9a 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -6,6 +6,7 @@ from twisted.internet import reactor from twisted.internet.defer import ( inlineCallbacks, DeferredQueue, Deferred, succeed) +from twisted.internet.task import Clock from twisted.web.client import HTTPConnectionPool from twisted.python import log @@ -47,6 +48,7 @@ def setUp(self): 'test' ) self.consular.set_debug(True) + self.consular.clock = Clock() # spin up a site so we can test it, pretty sure Klein has better # ways of doing this but they're not documented anywhere. @@ -267,6 +269,42 @@ def test_TASK_KILLED(self): 'status': 'ok' }) + @inlineCallbacks + def test_schedule_sync(self): + """ + When Consular is set to schedule syncs, a sync should occur right away + and further syncs should occur after the correct delay. + """ + lc, d = self.consular.schedule_sync(1) + + self.assertTrue(lc.running) + + # Consular should do the first sync right away + request = yield self.requests.get() + self.assertEqual(request['method'], 'GET') + self.assertEqual( + request['url'], + 'http://localhost:8080/v2/apps') + + # Return no apps... let's make this quick + request['deferred'].callback( + FakeResponse(200, [], json.dumps({'apps': []}))) + + # Advance the clock for the next sync + self.consular.clock.advance(1) + + request = yield self.requests.get() + self.assertEqual(request['method'], 'GET') + self.assertEqual( + request['url'], + 'http://localhost:8080/v2/apps') + + request['deferred'].callback( + FakeResponse(200, [], json.dumps({'apps': []}))) + + lc.stop() + yield d + @inlineCallbacks def test_register_with_marathon(self): d = self.consular.register_marathon_event_callback( From fd1542000bfcc0b3ce724338f7dc8c13b23bcca7 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Mon, 26 Oct 2015 12:08:53 +0200 Subject: [PATCH 064/111] Add a failing test --- consular/tests/test_main.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index b2a7f9a..f04f001 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -305,6 +305,42 @@ def test_schedule_sync(self): lc.stop() yield d + @inlineCallbacks + def test_schedule_sync_handles_server_errors(self): + """ + When Consular is set to schedule syncs, syncs should not be interrupted + due to errors in previously scheduled syncs. + """ + lc, d = self.consular.schedule_sync(1) + + self.assertTrue(lc.running) + + # Consular should do the first sync right away + request = yield self.requests.get() + self.assertEqual(request['method'], 'GET') + self.assertEqual( + request['url'], + 'http://localhost:8080/v2/apps') + + # Return a server error. + request['deferred'].callback(FakeResponse(500, [], 'Server error')) + + # Advance the clock for the next sync + self.consular.clock.advance(1) + + # The next sync should happen regardless of the previous server error + request = yield self.requests.get() + self.assertEqual(request['method'], 'GET') + self.assertEqual( + request['url'], + 'http://localhost:8080/v2/apps') + + request['deferred'].callback( + FakeResponse(200, [], json.dumps({'apps': []}))) + + lc.stop() + yield d + @inlineCallbacks def test_register_with_marathon(self): d = self.consular.register_marathon_event_callback( From d5717dd081fe0dea393f71858a6d1be3d6ba4eb2 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Mon, 26 Oct 2015 13:18:16 +0200 Subject: [PATCH 065/111] Catch errors during scheduled app syncs --- consular/main.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/consular/main.py b/consular/main.py index bd1b26a..da90892 100644 --- a/consular/main.py +++ b/consular/main.py @@ -122,10 +122,21 @@ def schedule_sync(self, interval, purge=False): A tuple of the LoopingCall object and the deferred created when it was started. """ - lc = LoopingCall(self.sync_apps, purge) + lc = LoopingCall(self._try_sync_apps, purge) lc.clock = self.clock return (lc, lc.start(interval, now=True)) + @inlineCallbacks + def _try_sync_apps(self, purge=False): + """ + Sync the apps, catching and logging any exception that occurs. + """ + try: + yield self.sync_apps(purge) + except Exception as e: + # TODO: More specialised exception handling. + log.msg('Error syncing apps: %s' % e) + @inlineCallbacks def register_marathon_event_callback(self, events_url): """ From 3049afb2f26b055b243a2fd6720a59684e6e7e37 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Mon, 26 Oct 2015 16:19:48 +0200 Subject: [PATCH 066/111] Clean up service register/deregister methods * Preparing for updating task registration --- consular/main.py | 65 ++++++++++++++++++------------------- consular/tests/test_main.py | 52 ++++++++++++++++++++++++----- 2 files changed, 75 insertions(+), 42 deletions(-) diff --git a/consular/main.py b/consular/main.py index da90892..50cb2ec 100644 --- a/consular/main.py +++ b/consular/main.py @@ -231,10 +231,7 @@ def update_task_running(self, request, event): return d def update_task_killed(self, request, event): - d = self.deregister_service( - get_agent_endpoint(event['host']), - get_app_name(event['appId']), - event['taskId']) + d = self.deregister_task_service(event['taskId'], event['host']) d.addCallback(lambda _: json.dumps({'status': 'ok'})) return d @@ -297,45 +294,51 @@ def _create_service_registration(self, app_id, service_id, address, port): } return registration - def register_service(self, agent_endpoint, - app_id, service_id, address, port): + def register_task_service(self, app_id, task_id, host, port): """ - Register a task in Marathon as a service in Consul + Register a Marathon task as a service in Consul. - :param str agent_endpoint: - The HTTP endpoint of where Consul on the Mesos worker machine - can be accessed. :param str app_id: - Marathon's App-id for the task. - :param str service_id: - The service-id to register it as in Consul. - :param str address: + The ID of the Marathon app that the task belongs to. + :param str task_id: + The ID of the task, this will be used as the Consul service ID. + :param str host: The host address of the machine the task is running on. :param int port: The port number the task can be accessed on on the host machine. """ + agent_endpoint = get_agent_endpoint(host) log.msg('Registering %s at %s with %s at %s:%s.' % ( - app_id, agent_endpoint, service_id, address, port)) - registration = self._create_service_registration(app_id, service_id, - address, port) + app_id, agent_endpoint, task_id, host, port)) + registration = self._create_service_registration(app_id, task_id, + host, port) return self.consul_client.register_agent_service( agent_endpoint, registration) - def deregister_service(self, agent_endpoint, app_id, service_id): + def deregister_task_service(self, task_id, host): """ - Deregister a service from Consul + Deregister a Marathon task's service from Consul. + + :param str task_id: + The ID of the task, this will be used as the Consul service ID. + :param str host: + The host address of the machine the task is running on. + """ + return self.deregister_consul_service( + get_agent_endpoint(host), task_id) + + def deregister_consul_service(self, agent_endpoint, service_id): + """ + Deregister a service from a Consul agent. :param str agent_endpoint: - The HTTP endpoint of where Consul on the Mesos worker machine - can be accessed. - :param str app_id: - Marathon's App-id for the task. + The HTTP endpoint of the Consul agent. :param str service_id: - The service-id to register it as in Consul. + The ID of the Consul service to be deregistered. """ - log.msg('Deregistering %s at %s with %s' % ( - app_id, agent_endpoint, service_id,)) + log.msg('Deregistering service with ID "%s" at Consul endpoint %s ' % ( + service_id, agent_endpoint,)) return self.consul_client.deregister_agent_service( agent_endpoint, service_id) @@ -501,12 +504,8 @@ def _consul_key_to_marathon_label_key(self, consul_key): def sync_app_tasks(self, app): tasks = yield self.marathon_client.get_app_tasks(app['id']) for task in tasks: - yield self.sync_app_task(app, task) - - def sync_app_task(self, app, task): - return self.register_service( - get_agent_endpoint(task['host']), app['id'], task['id'], - task['host'], task['ports'][0]) + yield self.register_task_service( + app['id'], task['id'], task['host'], task['ports'][0]) @inlineCallbacks def purge_dead_app_labels(self, apps): @@ -606,7 +605,7 @@ def purge_service_if_dead(self, agent_endpoint, app_id, consul_task_ids): # Deregister the remaining old services for service_id in service_ids: - yield self.deregister_service(agent_endpoint, app_id, service_id) + yield self.deregister_consul_service(agent_endpoint, service_id) def _filter_marathon_tasks(self, marathon_tasks, consul_service_ids): if not marathon_tasks: diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index f04f001..10194d2 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -402,10 +402,29 @@ def test_register_with_marathon_unexpected_response(self): 'http://localhost:8080/v2/eventSubscriptions') @inlineCallbacks - def test_sync_app_task(self): - app = {'id': '/my-app'} - task = {'id': 'my-task-id', 'host': '0.0.0.0', 'ports': [1234]} - d = self.consular.sync_app_task(app, task) + def test_sync_app_tasks(self): + """ + When syncing an app with a task, Consul is updated with a service entry + for the task. + """ + d = self.consular.sync_app_tasks({'id': '/my-app'}) + + # First Consular fetches the tasks for the app + marathon_request = yield self.requests.get() + self.assertEqual(marathon_request['method'], 'GET') + self.assertEqual( + marathon_request['url'], + 'http://localhost:8080/v2/apps/my-app/tasks') + + # Respond with one task + marathon_request['deferred'].callback( + FakeResponse(200, [], json.dumps({ + 'tasks': [ + {'id': 'my-task-id', 'host': '0.0.0.0', 'ports': [1234]} + ]})) + ) + + # Consular should register the task in Consul consul_request = yield self.requests.get() self.assertEqual( consul_request['url'], @@ -431,9 +450,24 @@ def test_sync_app_task_grouped(self): When syncing an app in a group with a task, Consul is updated with a service entry for the task. """ - app = {'id': '/my-group/my-app'} - task = {'id': 'my-task-id', 'host': '0.0.0.0', 'ports': [1234]} - d = self.consular.sync_app_task(app, task) + d = self.consular.sync_app_tasks({'id': '/my-group/my-app'}) + + # First Consular fetches the tasks for the app + marathon_request = yield self.requests.get() + self.assertEqual(marathon_request['method'], 'GET') + self.assertEqual( + marathon_request['url'], + 'http://localhost:8080/v2/apps/my-group/my-app/tasks') + + # Respond with one task + marathon_request['deferred'].callback( + FakeResponse(200, [], json.dumps({ + 'tasks': [ + {'id': 'my-task-id', 'host': '0.0.0.0', 'ports': [1234]} + ]})) + ) + + # Consular should register the task in Consul consul_request = yield self.requests.get() self.assertEqual( consul_request['url'], @@ -1036,8 +1070,8 @@ def test_purge_dead_app_labels_forbidden(self): @inlineCallbacks def test_fallback_to_main_consul(self): self.consular.consul_client.enable_fallback = True - self.consular.register_service( - 'http://foo:8500', '/app_id', 'service_id', 'foo', 1234) + self.consular.register_task_service( + '/app_id', 'service_id', 'foo', 1234) request = yield self.requests.get() self.assertEqual( request['url'], From 765c6fa3ceadc8e5c20ef59a663c16c1286a9fa2 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Mon, 26 Oct 2015 16:46:35 +0200 Subject: [PATCH 067/111] Improve new app event handling * Don't sync the whole app, just the task and the app labels. --- consular/main.py | 17 ++++++++----- consular/tests/test_main.py | 50 ++++++++++++++----------------------- 2 files changed, 30 insertions(+), 37 deletions(-) diff --git a/consular/main.py b/consular/main.py index 50cb2ec..7ba7979 100644 --- a/consular/main.py +++ b/consular/main.py @@ -222,13 +222,18 @@ def noop(self, request, event): 'status': 'ok' })) + @inlineCallbacks def update_task_running(self, request, event): - # NOTE: Marathon sends a list of ports, I don't know yet when & if - # there are multiple values in that list. - d = self.marathon_client.get_app(event['appId']) - d.addCallback(lambda app: self.sync_app(app)) - d.addCallback(lambda _: json.dumps({'status': 'ok'})) - return d + """ Use a running event to register a new Consul service. """ + # Register the task as a service + yield self.register_task_service( + event['appId'], event['taskId'], event['host'], event['ports'][0]) + + # Sync the app labels in case they've changed or aren't stored yet + app = yield self.marathon_client.get_app(event['appId']) + yield self.sync_app_labels(app) + + returnValue(json.dumps({'status': 'ok'})) def update_task_killed(self, request, event): d = self.deregister_task_service(event['taskId'], event['host']) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 10194d2..8ef9a1c 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -187,6 +187,25 @@ def test_TASK_RUNNING(self): "version": "2014-04-04T06:26:23.051Z" }) + # Store the task as a service in Consul + consul_request = yield self.requests.get() + self.assertEqual(consul_request['method'], 'PUT') + self.assertEqual( + consul_request['url'], + 'http://slave-1234.acme.org:8500/v1/agent/service/register') + self.assertEqual(consul_request['data'], json.dumps({ + 'Name': 'my-app', + 'ID': 'my-app_0-1396592784349', + 'Address': 'slave-1234.acme.org', + 'Port': 31372, + 'Tags': [ + 'consular-reg-id=test', + 'consular-app-id=/my-app', + ], + })) + consul_request['deferred'].callback( + FakeResponse(200, [], json.dumps({}))) + # We should get the app info for the event marathon_app_request = yield self.requests.get() self.assertEqual(marathon_app_request['method'], 'GET') @@ -207,37 +226,6 @@ def test_TASK_RUNNING(self): consul_kv_request['deferred'].callback( FakeResponse(200, [], json.dumps([]))) - # Then we collect the tasks for the app - marathon_tasks_request = yield self.requests.get() - self.assertEqual(marathon_tasks_request['method'], 'GET') - self.assertEqual(marathon_tasks_request['url'], - 'http://localhost:8080/v2/apps/my-app/tasks') - marathon_tasks_request['deferred'].callback( - FakeResponse(200, [], json.dumps({ - 'tasks': [{ - 'id': 'my-app_0-1396592784349', - 'host': 'slave-1234.acme.org', - 'ports': [31372], - }] - }))) - - request = yield self.requests.get() - self.assertEqual(request['method'], 'PUT') - self.assertEqual( - request['url'], - 'http://slave-1234.acme.org:8500/v1/agent/service/register') - self.assertEqual(request['data'], json.dumps({ - 'Name': 'my-app', - 'ID': 'my-app_0-1396592784349', - 'Address': 'slave-1234.acme.org', - 'Port': 31372, - 'Tags': [ - 'consular-reg-id=test', - 'consular-app-id=/my-app', - ], - })) - request['deferred'].callback( - FakeResponse(200, [], json.dumps({}))) response = yield d self.assertEqual((yield response.json()), { 'status': 'ok' From 0ecbd882311b31671e751c85aa9e70aa3b187d83 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Mon, 26 Oct 2015 16:54:44 +0200 Subject: [PATCH 068/111] Defensively code for disappearing apps --- consular/main.py | 13 ++++++++-- consular/tests/test_main.py | 48 +++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/consular/main.py b/consular/main.py index 7ba7979..d5c5ca1 100644 --- a/consular/main.py +++ b/consular/main.py @@ -230,8 +230,17 @@ def update_task_running(self, request, event): event['appId'], event['taskId'], event['host'], event['ports'][0]) # Sync the app labels in case they've changed or aren't stored yet - app = yield self.marathon_client.get_app(event['appId']) - yield self.sync_app_labels(app) + app = yield handle_not_found_error( + self.marathon_client.get_app, event['appId']) + + # The app could have disappeared in this time if it was destroyed. If + # it has been destroyed, do nothing and wait for the TASK_KILLED event + # to clear it. + if app is not None: + yield self.sync_app_labels(app) + else: + log.msg('Warning. App with ID "%s" could not be found for new ' + 'task with ID "%s"' % (event['appId'], event['taskId'],)) returnValue(json.dumps({'status': 'ok'})) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 8ef9a1c..afb3830 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -231,6 +231,54 @@ def test_TASK_RUNNING(self): 'status': 'ok' }) + @inlineCallbacks + def test_TASK_RUNNING_app_not_found(self): + d = self.request('POST', '/events', { + "eventType": "status_update_event", + "timestamp": "2014-03-01T23:29:30.158Z", + "slaveId": "20140909-054127-177048842-5050-1494-0", + "taskId": "my-app_0-1396592784349", + "taskStatus": "TASK_RUNNING", + "appId": "/my-app", + "host": "slave-1234.acme.org", + "ports": [31372], + "version": "2014-04-04T06:26:23.051Z" + }) + + # Store the task as a service in Consul + consul_request = yield self.requests.get() + self.assertEqual(consul_request['method'], 'PUT') + self.assertEqual( + consul_request['url'], + 'http://slave-1234.acme.org:8500/v1/agent/service/register') + self.assertEqual(consul_request['data'], json.dumps({ + 'Name': 'my-app', + 'ID': 'my-app_0-1396592784349', + 'Address': 'slave-1234.acme.org', + 'Port': 31372, + 'Tags': [ + 'consular-reg-id=test', + 'consular-app-id=/my-app', + ], + })) + consul_request['deferred'].callback( + FakeResponse(200, [], json.dumps({}))) + + # We try to get the app info for the event but the app is gone + marathon_app_request = yield self.requests.get() + self.assertEqual(marathon_app_request['method'], 'GET') + self.assertEqual(marathon_app_request['url'], + 'http://localhost:8080/v2/apps/my-app') + marathon_app_request['deferred'].callback( + FakeResponse(404, [], json.dumps({'message': 'Not found'}))) + + # So we do nothing... + + response = yield d + self.assertEqual((yield response.json()), { + 'status': 'ok' + }) + @inlineCallbacks def test_TASK_KILLED(self): d = self.request('POST', '/events', { From 88e6607887ffeb6a49e14674ccabad0aefcac351 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 27 Oct 2015 09:04:59 +0200 Subject: [PATCH 069/111] Add some failing tests --- consular/tests/test_main.py | 105 +++++++++++++++++++++++++++++++++++- 1 file changed, 104 insertions(+), 1 deletion(-) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index afb3830..7b1cce1 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -279,6 +279,67 @@ def test_TASK_RUNNING_app_not_found(self): 'status': 'ok' }) + @inlineCallbacks + def test_TASK_RUNNING_no_ports(self): + """ + When a TASK_RUNNING event is received from Marathon, and the task has + no ports, the task should be registered as a service in Consul. + """ + d = self.request('POST', '/events', { + "eventType": "status_update_event", + "timestamp": "2014-03-01T23:29:30.158Z", + "slaveId": "20140909-054127-177048842-5050-1494-0", + "taskId": "my-app_0-1396592784349", + "taskStatus": "TASK_RUNNING", + "appId": "/my-app", + "host": "slave-1234.acme.org", + "ports": [], + "version": "2014-04-04T06:26:23.051Z" + }) + + # Store the task as a service in Consul with no port + consul_request = yield self.requests.get() + self.assertEqual(consul_request['method'], 'PUT') + self.assertEqual( + consul_request['url'], + 'http://slave-1234.acme.org:8500/v1/agent/service/register') + self.assertEqual(consul_request['data'], json.dumps({ + 'Name': 'my-app', + 'ID': 'my-app_0-1396592784349', + 'Address': 'slave-1234.acme.org', + 'Tags': [ + 'consular-reg-id=test', + 'consular-app-id=/my-app', + ], + })) + consul_request['deferred'].callback( + FakeResponse(200, [], json.dumps({}))) + + # We should get the app info for the event + marathon_app_request = yield self.requests.get() + self.assertEqual(marathon_app_request['method'], 'GET') + self.assertEqual(marathon_app_request['url'], + 'http://localhost:8080/v2/apps/my-app') + marathon_app_request['deferred'].callback( + FakeResponse(200, [], json.dumps({ + 'app': { + 'id': '/my-app', + } + }))) + + # Check if any existing labels stored in Consul + consul_kv_request = yield self.requests.get() + self.assertEqual(consul_kv_request['method'], 'GET') + self.assertEqual(consul_kv_request['url'], + 'http://localhost:8500/v1/kv/consular/my-app?keys=') + consul_kv_request['deferred'].callback( + FakeResponse(200, [], json.dumps([]))) + + response = yield d + self.assertEqual((yield response.json()), { + 'status': 'ok' + }) + @inlineCallbacks def test_TASK_KILLED(self): d = self.request('POST', '/events', { @@ -481,7 +542,7 @@ def test_sync_app_tasks(self): yield d @inlineCallbacks - def test_sync_app_task_grouped(self): + def test_sync_app_tasks_grouped(self): """ When syncing an app in a group with a task, Consul is updated with a service entry for the task. @@ -523,6 +584,48 @@ def test_sync_app_task_grouped(self): FakeResponse(200, [], json.dumps({}))) yield d + @inlineCallbacks + def test_sync_app_tasks_no_ports(self): + """ + When syncing an app with a task with no ports, Consul is updated with a + service entry for the task. + """ + d = self.consular.sync_app_tasks({'id': '/my-app'}) + + # First Consular fetches the tasks for the app + marathon_request = yield self.requests.get() + self.assertEqual(marathon_request['method'], 'GET') + self.assertEqual( + marathon_request['url'], + 'http://localhost:8080/v2/apps/my-app/tasks') + + # Respond with one task + marathon_request['deferred'].callback( + FakeResponse(200, [], json.dumps({ + 'tasks': [ + {'id': 'my-task-id', 'host': '0.0.0.0', 'ports': []} + ]})) + ) + + # Consular should register the task in Consul with no port + consul_request = yield self.requests.get() + self.assertEqual( + consul_request['url'], + 'http://0.0.0.0:8500/v1/agent/service/register') + self.assertEqual(consul_request['data'], json.dumps({ + 'Name': 'my-app', + 'ID': 'my-task-id', + 'Address': '0.0.0.0', + 'Tags': [ + 'consular-reg-id=test', + 'consular-app-id=/my-app', + ], + })) + self.assertEqual(consul_request['method'], 'PUT') + consul_request['deferred'].callback( + FakeResponse(200, [], json.dumps({}))) + yield d + @inlineCallbacks def test_sync_app_labels(self): app = { From d8d3927e60261cc8ce0edec4bf8ffb63d395489e Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 27 Oct 2015 09:42:46 +0200 Subject: [PATCH 070/111] Support 0-1 ports for tasks/services * If 0 ports, don't specify a port for the Consul service * If more than 1 port, use the lowest port --- consular/main.py | 25 +++++++++++++++++++------ consular/tests/test_main.py | 2 +- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/consular/main.py b/consular/main.py index d5c5ca1..67db51a 100644 --- a/consular/main.py +++ b/consular/main.py @@ -227,7 +227,7 @@ def update_task_running(self, request, event): """ Use a running event to register a new Consul service. """ # Register the task as a service yield self.register_task_service( - event['appId'], event['taskId'], event['host'], event['ports'][0]) + event['appId'], event['taskId'], event['host'], event['ports']) # Sync the app labels in case they've changed or aren't stored yet app = yield handle_not_found_error( @@ -300,15 +300,17 @@ def _create_service_registration(self, app_id, service_id, address, port): 'Name': get_app_name(app_id), 'ID': service_id, 'Address': address, - 'Port': port, 'Tags': [ self.reg_id_tag(), self.app_id_tag(app_id), ] } + if port is not None: + registration['Port'] = port + return registration - def register_task_service(self, app_id, task_id, host, port): + def register_task_service(self, app_id, task_id, host, ports): """ Register a Marathon task as a service in Consul. @@ -318,9 +320,20 @@ def register_task_service(self, app_id, task_id, host, port): The ID of the task, this will be used as the Consul service ID. :param str host: The host address of the machine the task is running on. - :param int port: - The port number the task can be accessed on on the host machine. + :param list ports: + The port numbers the task can be accessed on on the host machine. """ + if not ports: + port = None + elif len(ports) == 1: + [port] = ports + else: + # TODO: Support multiple ports (issue #29) + port = sorted(ports)[0] + log.msg('Warning. %d ports found for app "%s". Consular currently ' + 'only supports a single port. Only the lowest port (%s) ' + 'will be used.' % (len(ports), app_id, port,)) + agent_endpoint = get_agent_endpoint(host) log.msg('Registering %s at %s with %s at %s:%s.' % ( app_id, agent_endpoint, task_id, host, port)) @@ -519,7 +532,7 @@ def sync_app_tasks(self, app): tasks = yield self.marathon_client.get_app_tasks(app['id']) for task in tasks: yield self.register_task_service( - app['id'], task['id'], task['host'], task['ports'][0]) + app['id'], task['id'], task['host'], task['ports']) @inlineCallbacks def purge_dead_app_labels(self, apps): diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 7b1cce1..98d6196 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -1210,7 +1210,7 @@ def test_purge_dead_app_labels_forbidden(self): def test_fallback_to_main_consul(self): self.consular.consul_client.enable_fallback = True self.consular.register_task_service( - '/app_id', 'service_id', 'foo', 1234) + '/app_id', 'service_id', 'foo', [1234]) request = yield self.requests.get() self.assertEqual( request['url'], From f7c360ca6cc0d7e8bc2f63edd2bff400d1286d40 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 27 Oct 2015 09:54:32 +0200 Subject: [PATCH 071/111] Add tests for multiple ports --- consular/tests/test_main.py | 110 ++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 98d6196..de2efaf 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -340,6 +340,69 @@ def test_TASK_RUNNING_no_ports(self): 'status': 'ok' }) + @inlineCallbacks + def test_TASK_RUNNING_multiple_ports(self): + """ + When a TASK_RUNNING event is received from Marathon, and the task has + multiple ports, the task should be registered as a service in Consul + with the lowest port. + """ + d = self.request('POST', '/events', { + "eventType": "status_update_event", + "timestamp": "2014-03-01T23:29:30.158Z", + "slaveId": "20140909-054127-177048842-5050-1494-0", + "taskId": "my-app_0-1396592784349", + "taskStatus": "TASK_RUNNING", + "appId": "/my-app", + "host": "slave-1234.acme.org", + "ports": [4567, 1234, 6789], + "version": "2014-04-04T06:26:23.051Z" + }) + + # Store the task as a service in Consul with the lowest port + consul_request = yield self.requests.get() + self.assertEqual(consul_request['method'], 'PUT') + self.assertEqual( + consul_request['url'], + 'http://slave-1234.acme.org:8500/v1/agent/service/register') + self.assertEqual(consul_request['data'], json.dumps({ + 'Name': 'my-app', + 'ID': 'my-app_0-1396592784349', + 'Address': 'slave-1234.acme.org', + 'Port': 1234, + 'Tags': [ + 'consular-reg-id=test', + 'consular-app-id=/my-app', + ], + })) + consul_request['deferred'].callback( + FakeResponse(200, [], json.dumps({}))) + + # We should get the app info for the event + marathon_app_request = yield self.requests.get() + self.assertEqual(marathon_app_request['method'], 'GET') + self.assertEqual(marathon_app_request['url'], + 'http://localhost:8080/v2/apps/my-app') + marathon_app_request['deferred'].callback( + FakeResponse(200, [], json.dumps({ + 'app': { + 'id': '/my-app', + } + }))) + + # Check if any existing labels stored in Consul + consul_kv_request = yield self.requests.get() + self.assertEqual(consul_kv_request['method'], 'GET') + self.assertEqual(consul_kv_request['url'], + 'http://localhost:8500/v1/kv/consular/my-app?keys=') + consul_kv_request['deferred'].callback( + FakeResponse(200, [], json.dumps([]))) + + response = yield d + self.assertEqual((yield response.json()), { + 'status': 'ok' + }) + @inlineCallbacks def test_TASK_KILLED(self): d = self.request('POST', '/events', { @@ -626,6 +689,53 @@ def test_sync_app_tasks_no_ports(self): FakeResponse(200, [], json.dumps({}))) yield d + @inlineCallbacks + def test_sync_app_tasks_multiple_ports(self): + """ + When syncing an app with a task with multiple ports, Consul is updated + with a service entry for the task with the lowest port. + """ + d = self.consular.sync_app_tasks({'id': '/my-app'}) + + # First Consular fetches the tasks for the app + marathon_request = yield self.requests.get() + self.assertEqual(marathon_request['method'], 'GET') + self.assertEqual( + marathon_request['url'], + 'http://localhost:8080/v2/apps/my-app/tasks') + + # Respond with one task + marathon_request['deferred'].callback( + FakeResponse(200, [], json.dumps({ + 'tasks': [ + { + 'id': 'my-task-id', + 'host': '0.0.0.0', + 'ports': [4567, 1234, 6789] + } + ]})) + ) + + # Consular should register the task in Consul with the lowest port + consul_request = yield self.requests.get() + self.assertEqual( + consul_request['url'], + 'http://0.0.0.0:8500/v1/agent/service/register') + self.assertEqual(consul_request['data'], json.dumps({ + 'Name': 'my-app', + 'ID': 'my-task-id', + 'Address': '0.0.0.0', + 'Port': 1234, + 'Tags': [ + 'consular-reg-id=test', + 'consular-app-id=/my-app', + ], + })) + self.assertEqual(consul_request['method'], 'PUT') + consul_request['deferred'].callback( + FakeResponse(200, [], json.dumps({}))) + yield d + @inlineCallbacks def test_sync_app_labels(self): app = { From 2e17ad030f5536e7e96041f8dfba72764871210a Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 27 Oct 2015 10:31:53 +0200 Subject: [PATCH 072/111] Compare JSON dicts in tests instead of strings * Fixes pypy tests also just a better way to do things --- consular/tests/test_main.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index de2efaf..2270c70 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -193,7 +193,7 @@ def test_TASK_RUNNING(self): self.assertEqual( consul_request['url'], 'http://slave-1234.acme.org:8500/v1/agent/service/register') - self.assertEqual(consul_request['data'], json.dumps({ + self.assertEqual(json.loads(consul_request['data']), { 'Name': 'my-app', 'ID': 'my-app_0-1396592784349', 'Address': 'slave-1234.acme.org', @@ -202,7 +202,7 @@ def test_TASK_RUNNING(self): 'consular-reg-id=test', 'consular-app-id=/my-app', ], - })) + }) consul_request['deferred'].callback( FakeResponse(200, [], json.dumps({}))) @@ -251,7 +251,7 @@ def test_TASK_RUNNING_app_not_found(self): self.assertEqual( consul_request['url'], 'http://slave-1234.acme.org:8500/v1/agent/service/register') - self.assertEqual(consul_request['data'], json.dumps({ + self.assertEqual(json.loads(consul_request['data']), { 'Name': 'my-app', 'ID': 'my-app_0-1396592784349', 'Address': 'slave-1234.acme.org', @@ -260,7 +260,7 @@ def test_TASK_RUNNING_app_not_found(self): 'consular-reg-id=test', 'consular-app-id=/my-app', ], - })) + }) consul_request['deferred'].callback( FakeResponse(200, [], json.dumps({}))) @@ -303,7 +303,7 @@ def test_TASK_RUNNING_no_ports(self): self.assertEqual( consul_request['url'], 'http://slave-1234.acme.org:8500/v1/agent/service/register') - self.assertEqual(consul_request['data'], json.dumps({ + self.assertEqual(json.loads(consul_request['data']), { 'Name': 'my-app', 'ID': 'my-app_0-1396592784349', 'Address': 'slave-1234.acme.org', @@ -311,7 +311,7 @@ def test_TASK_RUNNING_no_ports(self): 'consular-reg-id=test', 'consular-app-id=/my-app', ], - })) + }) consul_request['deferred'].callback( FakeResponse(200, [], json.dumps({}))) @@ -365,7 +365,7 @@ def test_TASK_RUNNING_multiple_ports(self): self.assertEqual( consul_request['url'], 'http://slave-1234.acme.org:8500/v1/agent/service/register') - self.assertEqual(consul_request['data'], json.dumps({ + self.assertEqual(json.loads(consul_request['data']), { 'Name': 'my-app', 'ID': 'my-app_0-1396592784349', 'Address': 'slave-1234.acme.org', @@ -374,7 +374,7 @@ def test_TASK_RUNNING_multiple_ports(self): 'consular-reg-id=test', 'consular-app-id=/my-app', ], - })) + }) consul_request['deferred'].callback( FakeResponse(200, [], json.dumps({}))) @@ -589,7 +589,7 @@ def test_sync_app_tasks(self): self.assertEqual( consul_request['url'], 'http://0.0.0.0:8500/v1/agent/service/register') - self.assertEqual(consul_request['data'], json.dumps({ + self.assertEqual(json.loads(consul_request['data']), { 'Name': 'my-app', 'ID': 'my-task-id', 'Address': '0.0.0.0', @@ -598,7 +598,7 @@ def test_sync_app_tasks(self): 'consular-reg-id=test', 'consular-app-id=/my-app', ], - })) + }) self.assertEqual(consul_request['method'], 'PUT') consul_request['deferred'].callback( FakeResponse(200, [], json.dumps({}))) @@ -632,7 +632,7 @@ def test_sync_app_tasks_grouped(self): self.assertEqual( consul_request['url'], 'http://0.0.0.0:8500/v1/agent/service/register') - self.assertEqual(consul_request['data'], json.dumps({ + self.assertEqual(json.loads(consul_request['data']), { 'Name': 'my-group-my-app', 'ID': 'my-task-id', 'Address': '0.0.0.0', @@ -641,7 +641,7 @@ def test_sync_app_tasks_grouped(self): 'consular-reg-id=test', 'consular-app-id=/my-group/my-app', ], - })) + }) self.assertEqual(consul_request['method'], 'PUT') consul_request['deferred'].callback( FakeResponse(200, [], json.dumps({}))) @@ -675,7 +675,7 @@ def test_sync_app_tasks_no_ports(self): self.assertEqual( consul_request['url'], 'http://0.0.0.0:8500/v1/agent/service/register') - self.assertEqual(consul_request['data'], json.dumps({ + self.assertEqual(json.loads(consul_request['data']), { 'Name': 'my-app', 'ID': 'my-task-id', 'Address': '0.0.0.0', @@ -683,7 +683,7 @@ def test_sync_app_tasks_no_ports(self): 'consular-reg-id=test', 'consular-app-id=/my-app', ], - })) + }) self.assertEqual(consul_request['method'], 'PUT') consul_request['deferred'].callback( FakeResponse(200, [], json.dumps({}))) @@ -721,7 +721,7 @@ def test_sync_app_tasks_multiple_ports(self): self.assertEqual( consul_request['url'], 'http://0.0.0.0:8500/v1/agent/service/register') - self.assertEqual(consul_request['data'], json.dumps({ + self.assertEqual(json.loads(consul_request['data']), { 'Name': 'my-app', 'ID': 'my-task-id', 'Address': '0.0.0.0', @@ -730,7 +730,7 @@ def test_sync_app_tasks_multiple_ports(self): 'consular-reg-id=test', 'consular-app-id=/my-app', ], - })) + }) self.assertEqual(consul_request['method'], 'PUT') consul_request['deferred'].callback( FakeResponse(200, [], json.dumps({}))) @@ -1334,7 +1334,7 @@ def test_fallback_to_main_consul(self): self.assertEqual( fallback_request['url'], 'http://localhost:8500/v1/agent/service/register') - self.assertEqual(fallback_request['data'], json.dumps({ + self.assertEqual(json.loads(fallback_request['data']), { 'Name': 'app_id', 'ID': 'service_id', 'Address': 'foo', @@ -1343,4 +1343,4 @@ def test_fallback_to_main_consul(self): 'consular-reg-id=test', 'consular-app-id=/app_id', ], - })) + }) From 202575aa34f754ed91a0bba11db12ab6f6b60395 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 27 Oct 2015 10:33:15 +0200 Subject: [PATCH 073/111] Add .cache directory to gitignore for pytest 2.8 --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 07458b6..6e42562 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ _trial_temp/ /docs/_build htmlcov/ +.cache/ From 4b09062de687fe387fc697aacb8f0472eb88f0d4 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Wed, 28 Oct 2015 09:24:09 +0200 Subject: [PATCH 074/111] Use min() instead of sorted()[0] --- consular/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consular/main.py b/consular/main.py index 67db51a..1429328 100644 --- a/consular/main.py +++ b/consular/main.py @@ -329,7 +329,7 @@ def register_task_service(self, app_id, task_id, host, ports): [port] = ports else: # TODO: Support multiple ports (issue #29) - port = sorted(ports)[0] + port = min(ports) log.msg('Warning. %d ports found for app "%s". Consular currently ' 'only supports a single port. Only the lowest port (%s) ' 'will be used.' % (len(ports), app_id, port,)) From a65e716e3db9ebfb48097b0fc4fb1181edd8c2d0 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Wed, 28 Oct 2015 09:35:52 +0200 Subject: [PATCH 075/111] Bump the version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 524cb55..26aaba0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.1 +1.2.0 From 96ae5e4c258e87c41596cd079375de6ed9ea4caa Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 16 Feb 2016 12:05:48 +0200 Subject: [PATCH 076/111] Make client agent configurable --- consular/clients.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/consular/clients.py b/consular/clients.py index 7cbf17e..9678708 100644 --- a/consular/clients.py +++ b/consular/clients.py @@ -16,6 +16,7 @@ class JsonClient(object): debug = False clock = reactor timeout = 5 + agent = None requester = lambda self, *a, **kw: treq.request(*a, **kw) def __init__(self, endpoint): @@ -66,6 +67,7 @@ def request(self, method, path, endpoint=None, json_data=None, **kwargs): }, 'data': data, 'pool': self.pool, + 'agent': self.agent, 'timeout': self.timeout } requester_kwargs.update(kwargs) From 959d0a2d2d213cc2893176049bdf4040ab9fac24 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 16 Feb 2016 12:06:22 +0200 Subject: [PATCH 077/111] Add initial tests for JsonClient using txfake --- consular/tests/test_clients.py | 174 +++++++++++++++++++++++++++++++++ requirements-dev.txt | 1 + 2 files changed, 175 insertions(+) create mode 100644 consular/tests/test_clients.py diff --git a/consular/tests/test_clients.py b/consular/tests/test_clients.py new file mode 100644 index 0000000..1746554 --- /dev/null +++ b/consular/tests/test_clients.py @@ -0,0 +1,174 @@ +import json + +from twisted.internet.defer import inlineCallbacks, DeferredQueue +from twisted.trial.unittest import TestCase +from twisted.web.server import NOT_DONE_YET + +from txfake import FakeHttpServer +from txfake.fake_connection import wait0 + +from consular.clients import ( + ConsulClient, HTTPError, JsonClient, MarathonClient) + + +class JsonClientTestBase(TestCase): + def setUp(self): + self.client = self.get_client() + self.requests = DeferredQueue() + self.fake_server = FakeHttpServer(self.handle_request) + + self.client.agent = self.fake_server.get_agent() + + def handle_request(self, request): + self.requests.put(request) + return NOT_DONE_YET + + def get_client(self): + """To be implemented by subclass""" + + def write_json_response(self, request, json_data, response_code=200, + headers={'Content-Type': 'application/json'}): + request.setResponseCode(response_code) + for name, value in headers.items(): + request.setHeader(name, value) + request.write(json.dumps(json_data)) + request.finish() + + +class JsonClientTest(JsonClientTestBase): + + def get_client(self): + return JsonClient('http://localhost:8000') + + @inlineCallbacks + def test_request(self): + """ + When a request is made, it should be made with the correct method, + address and headers, and should contain an empty body. The response + should be returned. + """ + d = self.client.request('GET', '/hello') + + request = yield self.requests.get() + self.assertEqual(request.method, 'GET') + self.assertEqual(request.uri, 'http://localhost:8000/hello') + self.assertEqual(request.getHeader('content-type'), 'application/json') + self.assertEqual(request.getHeader('accept'), 'application/json') + self.assertEqual(request.content.read(), '') + + request.setResponseCode(200) + request.write('hi\n') + request.finish() + + response = yield d + text = yield response.text() + self.assertEqual(text, 'hi\n') + + @inlineCallbacks + def test_request_json_data(self): + """ + When a request is made with the json_data parameter set, that data + should be sent as JSON. + """ + d = self.client.request('GET', '/hello', json_data={'test': 'hello'}) + + request = yield self.requests.get() + self.assertEqual(request.method, 'GET') + self.assertEqual(request.uri, 'http://localhost:8000/hello') + self.assertEqual(json.load(request.content), {'test': 'hello'}) + + request.setResponseCode(200) + request.finish() + + yield d + + @inlineCallbacks + def test_request_endpoint(self): + """ + When a request is made with the endpoint parameter set, that parameter + should be used as the endpoint. + """ + d = self.client.request('GET', '/hello', + endpoint='http://localhost:9000') + + request = yield self.requests.get() + self.assertEqual(request.method, 'GET') + self.assertEqual(request.uri, 'http://localhost:9000/hello') + + request.setResponseCode(200) + request.finish() + + yield d + + @inlineCallbacks + def test_get_json(self): + """ + When the get_json method is called, a GET request should be made and + the response should be deserialized from JSON. + """ + d = self.client.get_json('/hello') + + request = yield self.requests.get() + self.assertEqual(request.method, 'GET') + self.assertEqual(request.uri, 'http://localhost:8000/hello') + + request.setResponseCode(200) + request.write(json.dumps({'test': 'hello'})) + request.finish() + + res = yield d + self.assertEqual(res, {'test': 'hello'}) + + @inlineCallbacks + def test_client_error_response(self): + """ + When a request is made and a 4xx response code is returned, a HTTPError + should be raised to indicate a client error. + """ + d = self.client.request('GET', '/hello') + + request = yield self.requests.get() + self.assertEqual(request.method, 'GET') + self.assertEqual(request.uri, 'http://localhost:8000/hello') + + request.setResponseCode(403) + request.write('Unauthorized\n') + request.finish() + + yield wait0() + failure = self.failureResultOf(d, HTTPError) + self.assertEqual( + failure.getErrorMessage(), + '403 Client Error for url: http://localhost:8000/hello') + + @inlineCallbacks + def test_server_error_response(self): + """ + When a request is made and a 5xx response code is returned, a HTTPError + should be raised to indicate a server error. + """ + d = self.client.request('GET', '/hello') + + request = yield self.requests.get() + self.assertEqual(request.method, 'GET') + self.assertEqual(request.uri, 'http://localhost:8000/hello') + + request.setResponseCode(502) + request.write('Bad gateway\n') + request.finish() + + yield wait0() + failure = self.failureResultOf(d, HTTPError) + self.assertEqual( + failure.getErrorMessage(), + '502 Server Error for url: http://localhost:8000/hello') + + +class ConsulClientTest(JsonClientTestBase): + def get_client(self): + return ConsulClient('http://localhost:8500') + + +class MarathonClientTest(JsonClientTestBase): + def get_client(self): + return MarathonClient('http://localhost:8080') diff --git a/requirements-dev.txt b/requirements-dev.txt index c3ab0d5..2b7eac9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,3 +3,4 @@ pytest-coverage pytest-xdist flake8 sphinx +txfake From 3c68c2827150d6faa6bed905630d74c4085d20dd Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 16 Feb 2016 12:15:07 +0200 Subject: [PATCH 078/111] Fixes for new flake8 checks --- consular/clients.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/consular/clients.py b/consular/clients.py index 9678708..372b190 100644 --- a/consular/clients.py +++ b/consular/clients.py @@ -1,5 +1,6 @@ from urllib import quote, urlencode import json +import treq from twisted.internet import reactor from twisted.python import log @@ -9,15 +10,12 @@ # Twisted's default HTTP11 client factory is way too verbose client._HTTP11ClientFactory.noisy = False -import treq - class JsonClient(object): debug = False clock = reactor timeout = 5 agent = None - requester = lambda self, *a, **kw: treq.request(*a, **kw) def __init__(self, endpoint): """ @@ -26,6 +24,9 @@ def __init__(self, endpoint): self.endpoint = endpoint self.pool = client.HTTPConnectionPool(self.clock, persistent=False) + def requester(self, *args, **kwargs): + return treq.request(*args, **kwargs) + def _log_http_response(self, response, method, path, data): log.msg('%s %s with %s returned: %s' % ( method, path, data, response.code)) From dfb1b4c13903bd19d17d89263038f8dcecd81961 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 16 Feb 2016 13:46:49 +0200 Subject: [PATCH 079/111] JsonClientTestBase: raise error if unimplemented get_client --- consular/tests/test_clients.py | 1 + 1 file changed, 1 insertion(+) diff --git a/consular/tests/test_clients.py b/consular/tests/test_clients.py index 1746554..782e6f6 100644 --- a/consular/tests/test_clients.py +++ b/consular/tests/test_clients.py @@ -25,6 +25,7 @@ def handle_request(self, request): def get_client(self): """To be implemented by subclass""" + raise NotImplementedError() def write_json_response(self, request, json_data, response_code=200, headers={'Content-Type': 'application/json'}): From 7b14a1de1a4beafa00807df02991eeeeb14fa2cc Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 16 Feb 2016 13:55:02 +0200 Subject: [PATCH 080/111] Add uri helper method --- consular/tests/test_clients.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/consular/tests/test_clients.py b/consular/tests/test_clients.py index 782e6f6..69ab980 100644 --- a/consular/tests/test_clients.py +++ b/consular/tests/test_clients.py @@ -35,6 +35,9 @@ def write_json_response(self, request, json_data, response_code=200, request.write(json.dumps(json_data)) request.finish() + def uri(self, path): + return '%s%s' % (self.client.endpoint, path,) + class JsonClientTest(JsonClientTestBase): @@ -52,7 +55,7 @@ def test_request(self): request = yield self.requests.get() self.assertEqual(request.method, 'GET') - self.assertEqual(request.uri, 'http://localhost:8000/hello') + self.assertEqual(request.uri, self.uri('/hello')) self.assertEqual(request.getHeader('content-type'), 'application/json') self.assertEqual(request.getHeader('accept'), 'application/json') self.assertEqual(request.content.read(), '') @@ -75,7 +78,7 @@ def test_request_json_data(self): request = yield self.requests.get() self.assertEqual(request.method, 'GET') - self.assertEqual(request.uri, 'http://localhost:8000/hello') + self.assertEqual(request.uri, self.uri('/hello')) self.assertEqual(json.load(request.content), {'test': 'hello'}) request.setResponseCode(200) @@ -111,7 +114,7 @@ def test_get_json(self): request = yield self.requests.get() self.assertEqual(request.method, 'GET') - self.assertEqual(request.uri, 'http://localhost:8000/hello') + self.assertEqual(request.uri, self.uri('/hello')) request.setResponseCode(200) request.write(json.dumps({'test': 'hello'})) @@ -130,7 +133,7 @@ def test_client_error_response(self): request = yield self.requests.get() self.assertEqual(request.method, 'GET') - self.assertEqual(request.uri, 'http://localhost:8000/hello') + self.assertEqual(request.uri, self.uri('/hello')) request.setResponseCode(403) request.write('Unauthorized\n') @@ -140,7 +143,7 @@ def test_client_error_response(self): failure = self.failureResultOf(d, HTTPError) self.assertEqual( failure.getErrorMessage(), - '403 Client Error for url: http://localhost:8000/hello') + '403 Client Error for url: %s' % self.uri('/hello')) @inlineCallbacks def test_server_error_response(self): @@ -152,7 +155,7 @@ def test_server_error_response(self): request = yield self.requests.get() self.assertEqual(request.method, 'GET') - self.assertEqual(request.uri, 'http://localhost:8000/hello') + self.assertEqual(request.uri, self.uri('/hello')) request.setResponseCode(502) request.write('Bad gateway\n') @@ -162,7 +165,7 @@ def test_server_error_response(self): failure = self.failureResultOf(d, HTTPError) self.assertEqual( failure.getErrorMessage(), - '502 Server Error for url: http://localhost:8000/hello') + '502 Server Error for url: %s' % self.uri('/hello')) class ConsulClientTest(JsonClientTestBase): From 8b224043e69767dcdc401ae61dfc07123c768ab7 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 16 Feb 2016 14:16:32 +0200 Subject: [PATCH 081/111] Remove empty Consul/MarathonClient test cases for now --- consular/tests/test_clients.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/consular/tests/test_clients.py b/consular/tests/test_clients.py index 69ab980..ef58e08 100644 --- a/consular/tests/test_clients.py +++ b/consular/tests/test_clients.py @@ -166,13 +166,3 @@ def test_server_error_response(self): self.assertEqual( failure.getErrorMessage(), '502 Server Error for url: %s' % self.uri('/hello')) - - -class ConsulClientTest(JsonClientTestBase): - def get_client(self): - return ConsulClient('http://localhost:8500') - - -class MarathonClientTest(JsonClientTestBase): - def get_client(self): - return MarathonClient('http://localhost:8080') From f15e81709e8aa5160524bd8cd5a15794430a56f4 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 16 Feb 2016 14:19:08 +0200 Subject: [PATCH 082/111] Remove unneeded imports --- consular/tests/test_clients.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/consular/tests/test_clients.py b/consular/tests/test_clients.py index ef58e08..9aa292c 100644 --- a/consular/tests/test_clients.py +++ b/consular/tests/test_clients.py @@ -7,8 +7,7 @@ from txfake import FakeHttpServer from txfake.fake_connection import wait0 -from consular.clients import ( - ConsulClient, HTTPError, JsonClient, MarathonClient) +from consular.clients import HTTPError, JsonClient class JsonClientTestBase(TestCase): From 9ef456cb22655751f606f44f30b0228eb41f6fbf Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 16 Feb 2016 14:35:25 +0200 Subject: [PATCH 083/111] Move install requirements to setup.py and clean up Travis file --- .travis.yml | 7 +++---- requirements.txt | 6 ++---- setup.py | 11 ++++++----- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 544cb27..d491cae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,13 +6,12 @@ cache: directories: - $HOME/.cache/pip install: - - pip install "cryptography<=1.0" # NOTE: because pypy<2.5 on Travis - - pip install twine - - pip install coveralls - pip install --upgrade pip - - pip install flake8 + - pip install "cryptography<=1.0" # NOTE: because pypy<2.5 on Travis - pip install -r requirements-dev.txt - pip install -e . + - pip install coveralls + - pip install twine script: - flake8 consular - py.test consular --cov consular diff --git a/requirements.txt b/requirements.txt index 903109c..0b5bd7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,2 @@ -Twisted -Klein -treq -click +# Our dependencies are all specified in setup.py. +-e . diff --git a/setup.py b/setup.py index 43a96fb..bb91d83 100644 --- a/setup.py +++ b/setup.py @@ -6,9 +6,6 @@ with open(os.path.join(here, 'README.rst')) as f: README = f.read() -with open(os.path.join(here, 'requirements.txt')) as f: - requires = filter(None, f.readlines()) - with open(os.path.join(here, 'VERSION')) as f: version = f.read().strip() @@ -29,8 +26,12 @@ packages=find_packages(exclude=['docs']), include_package_data=True, zip_safe=False, - install_requires=requires, - tests_require=requires, + install_requires=[ + 'click', + 'Klein', + 'treq', + 'Twisted', + ], entry_points={ 'console_scripts': ['consular = consular.cli:main'], }) From 09609359de13bd01844c7fc823a9dc13cb36cd3a Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 16 Feb 2016 14:45:14 +0200 Subject: [PATCH 084/111] Use newer PyPy --- .travis.yml | 21 +++++++++++++++------ setup-pypy-travis.sh | 16 ++++++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 setup-pypy-travis.sh diff --git a/.travis.yml b/.travis.yml index d491cae..4f635b4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,22 +1,31 @@ language: python python: - "2.7" - - "pypy" +matrix: + include: + - python: "2.7" + - python: "pypy" + env: PYPY_VERSION="4.0.1" NO_COVERAGE=1 cache: directories: - $HOME/.cache/pip + - $HOME/downloads + +before_install: + # If necessary, set up an appropriate version of pypy. + - if [ ! -z "$PYPY_VERSION" ]; then source setup-pypy-travis.sh; fi + - if [ ! -z "$PYPY_VERSION" ]; then python --version 2>&1 | fgrep "PyPy $PYPY_VERSION"; fi install: - pip install --upgrade pip - - pip install "cryptography<=1.0" # NOTE: because pypy<2.5 on Travis - - pip install -r requirements-dev.txt - - pip install -e . + - pip install -r requirements.txt - pip install coveralls - pip install twine script: - flake8 consular - - py.test consular --cov consular + - if [ -z "$NO_COVERAGE" ]; then COVERAGE_OPT="--cov consular"; else COVERAGE_OPT=""; fi + - py.test consular $COVERAGE_OPT after_success: - - coveralls + - if [ -z "$NO_COVERAGE" ]; then coveralls; fi deploy: provider: pypi user: smn diff --git a/setup-pypy-travis.sh b/setup-pypy-travis.sh new file mode 100644 index 0000000..05caa67 --- /dev/null +++ b/setup-pypy-travis.sh @@ -0,0 +1,16 @@ +# NOTE: This script needs to be sourced so it can modify the environment. + +# Get out of the virtualenv we're in. +deactivate + +# Install pyenv. +curl -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash +export PATH="$HOME/.pyenv/bin:$PATH" +eval "$(pyenv init -)" +eval "$(pyenv virtualenv-init -)" + +# Install pypy and make a virtualenv for it. +pyenv install -s pypy-$PYPY_VERSION +pyenv global pypy-$PYPY_VERSION +virtualenv -p $(which python) ~/env-pypy-$PYPY_VERSION +source ~/env-pypy-$PYPY_VERSION/bin/activate From 9ab6173de7330250823bfb12585dfca2b9c86b9b Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 16 Feb 2016 14:48:29 +0200 Subject: [PATCH 085/111] Travis: Try fix requirements installation --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4f635b4..19f8295 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,8 @@ before_install: - if [ ! -z "$PYPY_VERSION" ]; then python --version 2>&1 | fgrep "PyPy $PYPY_VERSION"; fi install: - pip install --upgrade pip - - pip install -r requirements.txt + - pip install -e . + - pip install -r requirements-dev.txt - pip install coveralls - pip install twine script: From 28164840f9c08808877280360894d2a2825da3a8 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 16 Feb 2016 14:52:06 +0200 Subject: [PATCH 086/111] Travis: remove unnecesary python --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 19f8295..85b552a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ python: - "2.7" matrix: include: - - python: "2.7" - python: "pypy" env: PYPY_VERSION="4.0.1" NO_COVERAGE=1 cache: From 322a2d905adf240ead66750f5dafe4792d2d66fd Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 16 Feb 2016 16:51:49 +0200 Subject: [PATCH 087/111] Add some basic tests for MarathonClient --- consular/tests/test_clients.py | 256 ++++++++++++++++++++++++++++++++- 1 file changed, 255 insertions(+), 1 deletion(-) diff --git a/consular/tests/test_clients.py b/consular/tests/test_clients.py index 9aa292c..e381f9e 100644 --- a/consular/tests/test_clients.py +++ b/consular/tests/test_clients.py @@ -7,7 +7,7 @@ from txfake import FakeHttpServer from txfake.fake_connection import wait0 -from consular.clients import HTTPError, JsonClient +from consular.clients import HTTPError, JsonClient, MarathonClient class JsonClientTestBase(TestCase): @@ -165,3 +165,257 @@ def test_server_error_response(self): self.assertEqual( failure.getErrorMessage(), '502 Server Error for url: %s' % self.uri('/hello')) + + +class MarathonClientTest(JsonClientTestBase): + def get_client(self): + return MarathonClient('http://localhost:8080') + + @inlineCallbacks + def test_get_event_subscription(self): + """ + When we request event subscriptions from Marathon, we should receive a + list of callback URLs. + """ + d = self.client.get_event_subscriptions() + + request = yield self.requests.get() + self.assertEqual(request.method, 'GET') + self.assertEqual(request.uri, self.uri('/v2/eventSubscriptions')) + + self.write_json_response(request, { + 'callbackUrls': [ + 'http://localhost:7000/events?registration=localhost' + ] + }) + + res = yield d + self.assertEqual(res, [ + 'http://localhost:7000/events?registration=localhost' + ]) + + @inlineCallbacks + def test_post_event_subscription(self): + """ + When we post an event subscription with a callback URL, we should + return True for a 200/OK response from Marathon. + """ + d = self.client.post_event_subscription( + 'http://localhost:7000/events?registration=localhost') + + request = yield self.requests.get() + self.assertEqual(request.method, 'POST') + self.assertEqual(request.path, self.uri('/v2/eventSubscriptions')) + self.assertEqual(request.args, { + 'callbackUrl': [ + 'http://localhost:7000/events?registration=localhost' + ] + }) + + self.write_json_response(request, { + # TODO: Add check that callbackUrl is correct + 'callbackUrl': + 'http://localhost:7000/events?registration=localhost', + 'clientIp': '0:0:0:0:0:0:0:1', + 'eventType': 'subscribe_event' + }) + + res = yield d + self.assertEqual(res, True) + + @inlineCallbacks + def test_post_event_subscription_not_ok(self): + """ + When we post an event subscription with a callback URL, we should + return False for a non-200/OK response from Marathon. + """ + d = self.client.post_event_subscription( + 'http://localhost:7000/events?registration=localhost') + + request = yield self.requests.get() + self.assertEqual(request.method, 'POST') + self.assertEqual(request.path, self.uri('/v2/eventSubscriptions')) + self.assertEqual(request.args, { + 'callbackUrl': [ + 'http://localhost:7000/events?registration=localhost' + ] + }) + + self.write_json_response(request, {}, response_code=201) + + res = yield d + self.assertEqual(res, False) + + @inlineCallbacks + def test_get_apps(self): + """ + When we request the list of apps from Marathon, we should receive the + list of apps with some information. + """ + d = self.client.get_apps() + + request = yield self.requests.get() + self.assertEqual(request.method, 'GET') + self.assertEqual(request.uri, self.uri('/v2/apps')) + + apps = { + 'apps': [ + { + 'id': '/product/us-east/service/myapp', + 'cmd': 'env && sleep 60', + 'constraints': [ + [ + 'hostname', + 'UNIQUE', + '' + ] + ], + 'container': None, + 'cpus': 0.1, + 'env': { + 'LD_LIBRARY_PATH': '/usr/local/lib/myLib' + }, + 'executor': '', + 'instances': 3, + 'mem': 5.0, + 'ports': [ + 15092, + 14566 + ], + 'tasksRunning': 0, + 'tasksStaged': 1, + 'uris': [ + 'https://raw.github.com/mesosphere/marathon/master/' + 'README.md' + ], + 'version': '2014-03-01T23:42:20.938Z' + } + ] + } + self.write_json_response(request, apps) + + res = yield d + self.assertEqual(res, apps['apps']) + + @inlineCallbacks + def test_get_app(self): + """ + When we request information on a specific app from Marathon, we should + receive information on that app. + """ + d = self.client.get_app('/my-app') + + request = yield self.requests.get() + self.assertEqual(request.method, 'GET') + self.assertEqual(request.uri, self.uri('/v2/apps/my-app')) + + app = { + 'app': { + 'args': None, + 'backoffFactor': 1.15, + 'backoffSeconds': 1, + 'maxLaunchDelaySeconds': 3600, + 'cmd': 'python toggle.py $PORT0', + 'constraints': [], + 'container': None, + 'cpus': 0.2, + 'dependencies': [], + 'deployments': [ + { + 'id': '44c4ed48-ee53-4e0f-82dc-4df8b2a69057' + } + ], + 'disk': 0.0, + 'env': {}, + 'executor': '', + 'healthChecks': [ + { + 'command': None, + 'gracePeriodSeconds': 5, + 'intervalSeconds': 10, + 'maxConsecutiveFailures': 3, + 'path': '/health', + 'portIndex': 0, + 'protocol': 'HTTP', + 'timeoutSeconds': 10 + }, + { + 'command': None, + 'gracePeriodSeconds': 5, + 'intervalSeconds': 10, + 'maxConsecutiveFailures': 6, + 'path': '/machinehealth', + 'overridePort': 3333, + 'protocol': 'HTTP', + 'timeoutSeconds': 10 + } + ], + 'id': '/my-app', + 'instances': 2, + 'mem': 32.0, + 'ports': [ + 10000 + ], + 'requirePorts': False, + 'storeUrls': [], + 'upgradeStrategy': { + 'minimumHealthCapacity': 1.0 + }, + 'uris': [ + 'http://downloads.mesosphere.com/misc/toggle.tgz' + ], + 'user': None, + 'version': '2014-09-12T23:28:21.737Z', + 'versionInfo': { + 'lastConfigChangeAt': '2014-09-11T02:26:01.135Z', + 'lastScalingAt': '2014-09-12T23:28:21.737Z' + } + } + } + self.write_json_response(request, app) + + res = yield d + self.assertEqual(res, app['app']) + + @inlineCallbacks + def test_get_app_tasks(self): + """ + When we request the list of tasks for an app from Marathon, we should + receive a list of app tasks. + """ + d = self.client.get_app_tasks('/my-app') + + request = yield self.requests.get() + self.assertEqual(request.method, 'GET') + self.assertEqual(request.uri, self.uri('/v2/apps/my-app/tasks')) + + tasks = { + 'tasks': [ + { + 'host': 'agouti.local', + 'id': 'my-app_1-1396592790353', + 'ports': [ + 31336, + 31337 + ], + 'stagedAt': '2014-04-04T06:26:30.355Z', + 'startedAt': '2014-04-04T06:26:30.860Z', + 'version': '2014-04-04T06:26:23.051Z' + }, + { + 'host': 'agouti.local', + 'id': 'my-app_0-1396592784349', + 'ports': [ + 31382, + 31383 + ], + 'stagedAt': '2014-04-04T06:26:24.351Z', + 'startedAt': '2014-04-04T06:26:24.919Z', + 'version': '2014-04-04T06:26:23.051Z' + } + ] + } + self.write_json_response(request, tasks) + + res = yield d + self.assertEqual(res, tasks['tasks']) From 79facd28adda5f80203ad9e64d7f6e7c30f0b2cd Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 16 Feb 2016 16:53:25 +0200 Subject: [PATCH 088/111] MarathonClient: _basic_get_request -> get_json_field --- consular/clients.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/consular/clients.py b/consular/clients.py index 372b190..5a8ba8e 100644 --- a/consular/clients.py +++ b/consular/clients.py @@ -121,7 +121,7 @@ def __init__(self, message, response): class MarathonClient(JsonClient): - def _basic_get_request(self, path, field): + def get_json_field(self, path, field): """ Perform a GET request and get the contents of the JSON response. @@ -153,7 +153,7 @@ def get_event_subscriptions(self): Get the current Marathon event subscriptions, returning a list of callback URLs. """ - return self._basic_get_request( + return self.get_json_field( '/v2/eventSubscriptions', 'callbackUrls') def post_event_subscription(self, callback_url): @@ -171,20 +171,20 @@ def get_apps(self): Get the currently running Marathon apps, returning a list of app definitions. """ - return self._basic_get_request('/v2/apps', 'apps') + return self.get_json_field('/v2/apps', 'apps') def get_app(self, app_id): """ Get information about the app with the given app ID. """ - return self._basic_get_request('/v2/apps%s' % (app_id,), 'app') + return self.get_json_field('/v2/apps%s' % (app_id,), 'app') def get_app_tasks(self, app_id): """ Get the currently running tasks for the app with the given app ID, returning a list of task definitions. """ - return self._basic_get_request('/v2/apps%s/tasks' % (app_id,), 'tasks') + return self.get_json_field('/v2/apps%s/tasks' % (app_id,), 'tasks') class ConsulClient(JsonClient): From aafb8ef746bdf57736940b603d89ec149551b719 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 16 Feb 2016 17:06:05 +0200 Subject: [PATCH 089/111] Add get_json_field tests for MarathonClient --- consular/tests/test_clients.py | 43 ++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/consular/tests/test_clients.py b/consular/tests/test_clients.py index e381f9e..7d35ff4 100644 --- a/consular/tests/test_clients.py +++ b/consular/tests/test_clients.py @@ -171,6 +171,49 @@ class MarathonClientTest(JsonClientTestBase): def get_client(self): return MarathonClient('http://localhost:8080') + @inlineCallbacks + def test_get_json_field(self): + """ + When get_json_field is used to make a request, the response is + deserialized from JSON and the value of the specified field is + returned. + """ + d = self.client.get_json_field('/my-path', 'field-key') + + request = yield self.requests.get() + self.assertEqual(request.method, 'GET') + self.assertEqual(request.uri, self.uri('/my-path')) + + self.write_json_response(request, { + 'field-key': 'field-value', + 'other-field-key': 'do-not-care' + }) + + res = yield d + self.assertEqual(res, 'field-value') + + @inlineCallbacks + def test_get_json_field_missing(self): + """ + When get_json_field is used to make a request, the response is + deserialized from JSON and if the specified field is missing, an error + is raised. + """ + d = self.client.get_json_field('/my-path', 'field-key') + + request = yield self.requests.get() + self.assertEqual(request.method, 'GET') + self.assertEqual(request.uri, self.uri('/my-path')) + + self.write_json_response(request, {'other-field-key': 'do-not-care'}) + + yield wait0() + failure = self.failureResultOf(d, KeyError) + self.assertEqual( + failure.getErrorMessage(), + '\'Unable to get value for "field-key" from Marathon response: ' + '"{"other-field-key": "do-not-care"}"\'') + @inlineCallbacks def test_get_event_subscription(self): """ From 814ef3343dcc21c9894ea91518453e8895813931 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Wed, 17 Feb 2016 16:16:57 +0200 Subject: [PATCH 090/111] Basic Consul client tests --- consular/clients.py | 4 +- consular/tests/test_clients.py | 205 ++++++++++++++++++++++++++++++++- 2 files changed, 206 insertions(+), 3 deletions(-) diff --git a/consular/clients.py b/consular/clients.py index 5a8ba8e..c434b0e 100644 --- a/consular/clients.py +++ b/consular/clients.py @@ -269,8 +269,8 @@ def delete_kv_keys(self, key, recurse=False): :param: recurse: Whether or not to recursively delete all subpaths of the key. """ - return self.request('DELETE', '/v1/kv/%s%s' % ( - quote(key), '?recurse' if recurse else '',)) + query = '?recurse' if recurse else '' + return self.request('DELETE', '/v1/kv/%s%s' % (quote(key), query,)) def get_catalog_nodes(self): """ diff --git a/consular/tests/test_clients.py b/consular/tests/test_clients.py index 7d35ff4..48293f4 100644 --- a/consular/tests/test_clients.py +++ b/consular/tests/test_clients.py @@ -7,7 +7,8 @@ from txfake import FakeHttpServer from txfake.fake_connection import wait0 -from consular.clients import HTTPError, JsonClient, MarathonClient +from consular.clients import ( + ConsulClient, HTTPError, JsonClient, MarathonClient) class JsonClientTestBase(TestCase): @@ -462,3 +463,205 @@ def test_get_app_tasks(self): res = yield d self.assertEqual(res, tasks['tasks']) + + +class ConsulClientTest(JsonClientTestBase): + def get_client(self): + return ConsulClient('http://localhost:8500') + + @inlineCallbacks + def test_register_agent_service(self): + registration = { + 'ID': 'redis1', + 'Name': 'redis', + 'Tags': [ + 'master', + 'v1' + ], + 'Address': '127.0.0.1', + 'Port': 8000, + 'Check': { + 'Script': '/usr/local/bin/check_redis.py', + 'HTTP': 'http://localhost:5000/health', + 'Interval': '10s', + 'TTL': '15s' + } + } + d = self.client.register_agent_service('http://foo.example.com:8500', + registration) + + request = yield self.requests.get() + self.assertEqual(request.method, 'PUT') + self.assertEqual( + request.uri, + 'http://foo.example.com:8500/v1/agent/service/register') + self.assertEqual(json.load(request.content), registration) + + request.setResponseCode(200) + request.finish() + + yield d + + @inlineCallbacks + def test_deregister_agent_service(self): + d = self.client.deregister_agent_service('http://foo.example.com:8500', + 'redis1') + + request = yield self.requests.get() + self.assertEqual(request.method, 'PUT') + self.assertEqual( + request.uri, + 'http://foo.example.com:8500/v1/agent/service/deregister/redis1') + + request.setResponseCode(200) + request.finish() + + yield d + + @inlineCallbacks + def test_put_kv(self): + d = self.client.put_kv('foo', {'bar': 'baz'}) + + request = yield self.requests.get() + self.assertEqual(request.method, 'PUT') + self.assertEqual(request.uri, self.uri('/v1/kv/foo')) + self.assertEqual(json.load(request.content), {'bar': 'baz'}) + + request.setResponseCode(200) + request.write('true') + request.finish() + + res = yield d + json_res = yield res.json() + self.assertEqual(json_res, True) + + # TODO: Consul returns False. What should we do? + + @inlineCallbacks + def test_get_kv_keys(self): + d = self.client.get_kv_keys('foo') + + request = yield self.requests.get() + self.assertEqual(request.method, 'GET') + self.assertEqual(request.path, self.uri('/v1/kv/foo')) + self.assertEqual(request.args, { + 'keys': [''] + }) + + keys = [ + '/foo/bar', + '/foo/baz/boo' + ] + request.setResponseCode(200) + request.write(json.dumps(keys)) + request.finish() + + res = yield d + self.assertEqual(res, keys) + + @inlineCallbacks + def test_get_kv_keys_separator(self): + d = self.client.get_kv_keys('foo', separator='/') + + request = yield self.requests.get() + self.assertEqual(request.method, 'GET') + self.assertEqual(request.path, self.uri('/v1/kv/foo')) + self.assertEqual(request.args, { + 'keys': [''], + 'separator': ['/'] + }) + + keys = [ + '/foo/bar', + '/foo/baz/' + ] + request.setResponseCode(200) + request.write(json.dumps(keys)) + request.finish() + + res = yield d + self.assertEqual(res, keys) + + @inlineCallbacks + def test_delete_kv_keys(self): + d = self.client.delete_kv_keys('foo') + print d + #self.assertTrue(False) + + request = yield self.requests.get() + self.assertTrue(False) + self.assertEqual(request.method, 'PUT') + self.assertEqual(request.uri, self.uri('/v1/kv/foo')) + + request.setResponseCode(200) + request.finish() + + self.assertTrue(False) + + yield d +# +# @inlineCallbacks +# def test_delete_kv_keys_recursive(self): +# d = self.client.delete_kv_keys('foo', recurse=True) +# +# request = yield self.requests.get() +# self.assertEqual(request.method, 'DELETE') +# self.assertEqual(request.path, self.uri('/v1/kv/foo')) +# self.assertEqual(request.args, { +# 'recurse': [] +# }) +# +# request.setResponseCode(200) +# request.finish() +# +# yield d + + @inlineCallbacks + def test_get_catalog_nodes(self): + d = self.client.get_catalog_nodes() + + request = yield self.requests.get() + self.assertEqual(request.method, 'GET') + self.assertEqual(request.uri, self.uri('/v1/catalog/nodes')) + + nodes = [ + { + 'Node': 'baz', + 'Address': '10.1.10.11' + }, + { + 'Node': 'foobar', + 'Address': '10.1.10.12' + } + ] + request.setResponseCode(200) + request.write(json.dumps(nodes)) + request.finish() + + res = yield d + self.assertEqual(res, nodes) + + @inlineCallbacks + def test_get_agent_services(self): + d = self.client.get_agent_services('http://foo.example.com:8500') + + request = yield self.requests.get() + self.assertEqual(request.method, 'GET') + self.assertEqual(request.uri, + 'http://foo.example.com:8500/v1/agent/services') + + services = { + 'redis': { + 'ID': 'redis', + 'Service': 'redis', + 'Tags': None, + 'Address': 'http://foo.example.com', + 'Port': 8000 + } + } + request.setResponseCode(200) + request.write(json.dumps(services)) + request.finish() + + res = yield d + self.assertEqual(res, services) From 0064b64373adcf3c4c51412e057de84350903854 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Wed, 17 Feb 2016 17:18:15 +0200 Subject: [PATCH 091/111] Fix ConsulClient DELETE tests --- consular/tests/test_clients.py | 38 +++++++++++++++------------------- requirements-dev.txt | 2 +- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/consular/tests/test_clients.py b/consular/tests/test_clients.py index 48293f4..ff589f4 100644 --- a/consular/tests/test_clients.py +++ b/consular/tests/test_clients.py @@ -585,36 +585,32 @@ def test_get_kv_keys_separator(self): @inlineCallbacks def test_delete_kv_keys(self): d = self.client.delete_kv_keys('foo') - print d - #self.assertTrue(False) request = yield self.requests.get() - self.assertTrue(False) - self.assertEqual(request.method, 'PUT') + self.assertEqual(request.method, 'DELETE') self.assertEqual(request.uri, self.uri('/v1/kv/foo')) request.setResponseCode(200) request.finish() - self.assertTrue(False) + yield d + + @inlineCallbacks + def test_delete_kv_keys_recursive(self): + d = self.client.delete_kv_keys('foo', recurse=True) + + request = yield self.requests.get() + self.assertEqual(request.method, 'DELETE') + self.assertEqual(request.uri, self.uri('/v1/kv/foo?recurse')) + # request.args == {} :'( + # self.assertEqual(request.args, { + # 'recurse': [] + # }) + + request.setResponseCode(200) + request.finish() yield d -# -# @inlineCallbacks -# def test_delete_kv_keys_recursive(self): -# d = self.client.delete_kv_keys('foo', recurse=True) -# -# request = yield self.requests.get() -# self.assertEqual(request.method, 'DELETE') -# self.assertEqual(request.path, self.uri('/v1/kv/foo')) -# self.assertEqual(request.args, { -# 'recurse': [] -# }) -# -# request.setResponseCode(200) -# request.finish() -# -# yield d @inlineCallbacks def test_get_catalog_nodes(self): diff --git a/requirements-dev.txt b/requirements-dev.txt index 2b7eac9..8417d9e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,4 +3,4 @@ pytest-coverage pytest-xdist flake8 sphinx -txfake +txfake>=0.1.1 From 4ca661e720ccab318e8796a76c45ac39c2d813cb Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Wed, 17 Feb 2016 17:19:16 +0200 Subject: [PATCH 092/111] Remove 'consular' from setup.cfg --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 050d01b..e18604d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,2 @@ [pytest] -addopts = --doctest-modules --verbose --ignore=ve/ consular +addopts = --doctest-modules --verbose --ignore=ve/ From 83ff0488374cb2f6d0c67cfde80a5a67dd8a2479 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Thu, 18 Feb 2016 10:17:46 +0200 Subject: [PATCH 093/111] Add ConsulClient registration fallback test * Also make fallback method "private" --- consular/clients.py | 4 ++-- consular/tests/test_clients.py | 40 ++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/consular/clients.py b/consular/clients.py index c434b0e..d9e3b27 100644 --- a/consular/clients.py +++ b/consular/clients.py @@ -214,11 +214,11 @@ def register_agent_service(self, agent_endpoint, registration): endpoint=agent_endpoint, json_data=registration) if self.enable_fallback: - d.addErrback(self.register_agent_service_fallback, registration) + d.addErrback(self._register_agent_service_fallback, registration) return d - def register_agent_service_fallback(self, failure, registration): + def _register_agent_service_fallback(self, failure, registration): """ Fallback to the default agent endpoint (`self.endpoint`) to register a Consul service. diff --git a/consular/tests/test_clients.py b/consular/tests/test_clients.py index ff589f4..d0a82f9 100644 --- a/consular/tests/test_clients.py +++ b/consular/tests/test_clients.py @@ -502,6 +502,46 @@ def test_register_agent_service(self): yield d + @inlineCallbacks + def test_register_agent_service_fallback(self): + self.client.enable_fallback = True + # First try and do a regular registration + registration = { + 'ID': 'redis1', + 'Name': 'redis', + 'Tags': [ + 'master', + 'v1' + ], + 'Address': '127.0.0.1', + 'Port': 8000, + 'Check': { + 'Script': '/usr/local/bin/check_redis.py', + 'HTTP': 'http://localhost:5000/health', + 'Interval': '10s', + 'TTL': '15s' + } + } + d = self.client.register_agent_service('http://foo.example.com:8500', + registration) + + request = yield self.requests.get() + # Fail the request + request.setResponseCode(503) + request.write("Service unavailable\n") + request.finish() + + # Expect the request to fallback to the regular endpoint + request = yield self.requests.get() + self.assertEqual(request.method, 'PUT') + self.assertEqual(request.uri, self.uri('/v1/agent/service/register')) + self.assertEqual(json.load(request.content), registration) + + request.setResponseCode(200) + request.finish() + + yield d + @inlineCallbacks def test_deregister_agent_service(self): d = self.client.deregister_agent_service('http://foo.example.com:8500', From 588af01288ef9f7f38294c750e817f2e15c3d1af Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Thu, 18 Feb 2016 10:38:01 +0200 Subject: [PATCH 094/111] ConsulClient: Add test descriptions --- consular/tests/test_clients.py | 41 ++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/consular/tests/test_clients.py b/consular/tests/test_clients.py index d0a82f9..b15e839 100644 --- a/consular/tests/test_clients.py +++ b/consular/tests/test_clients.py @@ -471,6 +471,10 @@ def get_client(self): @inlineCallbacks def test_register_agent_service(self): + """ + When a service is registered with an agent, the registration JSON is + PUT to the correct address. + """ registration = { 'ID': 'redis1', 'Name': 'redis', @@ -504,6 +508,10 @@ def test_register_agent_service(self): @inlineCallbacks def test_register_agent_service_fallback(self): + """ + When a service is registered with an agent but the registration request + fails, the registration should fall back to the local Consul agent. + """ self.client.enable_fallback = True # First try and do a regular registration registration = { @@ -544,6 +552,10 @@ def test_register_agent_service_fallback(self): @inlineCallbacks def test_deregister_agent_service(self): + """ + When a service is deregistered, a PUT request is made to the correct + address. + """ d = self.client.deregister_agent_service('http://foo.example.com:8500', 'redis1') @@ -560,6 +572,10 @@ def test_deregister_agent_service(self): @inlineCallbacks def test_put_kv(self): + """ + When a value is put in the key/value store, a PUT request is made to + the correct address with the JSON data in the payload. + """ d = self.client.put_kv('foo', {'bar': 'baz'}) request = yield self.requests.get() @@ -579,6 +595,10 @@ def test_put_kv(self): @inlineCallbacks def test_get_kv_keys(self): + """ + When we get keys from the key/value store, a request is made to the + correct address and a list of keys is returned. + """ d = self.client.get_kv_keys('foo') request = yield self.requests.get() @@ -601,6 +621,11 @@ def test_get_kv_keys(self): @inlineCallbacks def test_get_kv_keys_separator(self): + """ + When we get keys from the key/value store and the "separator" parameter + is specified, a request is made to the correct address, the separator + is passed as a query parameter, and a list of keys is returned. + """ d = self.client.get_kv_keys('foo', separator='/') request = yield self.requests.get() @@ -624,6 +649,10 @@ def test_get_kv_keys_separator(self): @inlineCallbacks def test_delete_kv_keys(self): + """ + When we delete keys from the key/value store, a request is made to the + correct address. + """ d = self.client.delete_kv_keys('foo') request = yield self.requests.get() @@ -637,6 +666,10 @@ def test_delete_kv_keys(self): @inlineCallbacks def test_delete_kv_keys_recursive(self): + """ + When we delete keys from the key/value store recursively, a request is + made to the correct address with the "recurse" query parameter set. + """ d = self.client.delete_kv_keys('foo', recurse=True) request = yield self.requests.get() @@ -654,6 +687,10 @@ def test_delete_kv_keys_recursive(self): @inlineCallbacks def test_get_catalog_nodes(self): + """ + When we get the list of nodes from the catalog, a request is made to + the correct address and a list of nodes is returned. + """ d = self.client.get_catalog_nodes() request = yield self.requests.get() @@ -679,6 +716,10 @@ def test_get_catalog_nodes(self): @inlineCallbacks def test_get_agent_services(self): + """ + When we get the list of services from an agent, a request is made to + the correct address and a list of services is returned. + """ d = self.client.get_agent_services('http://foo.example.com:8500') request = yield self.requests.get() From 9b30832530af6fe9dd81a0808da0b0826832238e Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Thu, 18 Feb 2016 13:29:42 +0200 Subject: [PATCH 095/111] Use uritools for URI building and parsing --- consular/cli.py | 10 ++++------ consular/clients.py | 35 +++++++++++++++++----------------- consular/tests/test_clients.py | 27 +++++++++++++++++--------- consular/tests/test_main.py | 33 +++++++++++++++----------------- setup.py | 1 + 5 files changed, 56 insertions(+), 50 deletions(-) diff --git a/consular/cli.py b/consular/cli.py index 07abebb..d973330 100644 --- a/consular/cli.py +++ b/consular/cli.py @@ -1,7 +1,7 @@ import click import sys -from urllib import urlencode +from uritools import uricompose @click.command() @@ -63,11 +63,9 @@ def main(scheme, host, port, consular.set_debug(debug) consular.set_timeout(timeout) consular.fallback_timeout = fallback_timeout - events_url = "%s://%s:%s/events?%s" % ( - scheme, host, port, - urlencode({ - 'registration': registration_id, - })) + events_url = uricompose( + scheme=scheme, host=host, port=port, path='/events', + query={'registration': registration_id}) consular.register_marathon_event_callback(events_url) if sync_interval > 0: diff --git a/consular/clients.py b/consular/clients.py index d9e3b27..b0564f1 100644 --- a/consular/clients.py +++ b/consular/clients.py @@ -1,4 +1,3 @@ -from urllib import quote, urlencode import json import treq @@ -7,6 +6,8 @@ from twisted.web import client from twisted.web.http import OK +from uritools import uricompose, urisplit + # Twisted's default HTTP11 client factory is way too verbose client._HTTP11ClientFactory.noisy = False @@ -36,7 +37,8 @@ def _log_http_error(self, failure, url): log.err(failure, 'Error performing request to %s' % (url,)) return failure - def request(self, method, path, endpoint=None, json_data=None, **kwargs): + def request(self, method, path, query=None, endpoint=None, json_data=None, + **kwargs): """ Perform a request. A number of basic defaults are set on the request that make using a JSON API easier. These defaults can be overridden by @@ -45,8 +47,9 @@ def request(self, method, path, endpoint=None, json_data=None, **kwargs): :param: method: The HTTP method to use (example is `GET`). :param: path: - The URL path. This is appended to the endpoint and should start - with a '/' (example is `/v2/apps`). + The URL path (example is `/v2/apps`). + :param: query: + The URL query parameters as a dict. :param: endpoint: The URL endpoint to use. The default value is the endpoint this client was created with (`self.endpoint`) (example is @@ -58,7 +61,8 @@ def request(self, method, path, endpoint=None, json_data=None, **kwargs): Any other parameters that will be passed to `treq.request`, for example headers or parameters. """ - url = ('%s%s' % (endpoint or self.endpoint, path)).encode('utf-8') + scheme, authority = urisplit(endpoint or self.endpoint)[:2] + url = uricompose(scheme, authority, path, query) data = json.dumps(json_data) if json_data else None requester_kwargs = { @@ -81,11 +85,11 @@ def request(self, method, path, endpoint=None, json_data=None, **kwargs): d.addErrback(self._log_http_error, url) return d.addCallback(self._raise_for_status, url) - def get_json(self, path, **kwargs): + def get_json(self, path, query=None, **kwargs): """ Perform a GET request to the given path and return the JSON response. """ - d = self.request('GET', path, **kwargs) + d = self.request('GET', path, query, **kwargs) return d.addCallback(lambda response: response.json()) def _raise_for_status(self, response, url): @@ -161,9 +165,7 @@ def post_event_subscription(self, callback_url): Post a new Marathon event subscription with the given callback URL. """ d = self.request( - 'POST', '/v2/eventSubscriptions?%s' % urlencode({ - 'callbackUrl': callback_url, - })) + 'POST', '/v2/eventSubscriptions', {'callbackUrl': callback_url}) return d.addCallback(lambda response: response.code == OK) def get_apps(self): @@ -241,7 +243,7 @@ def put_kv(self, key, value): Put a key/value in Consul's k/v store. """ return self.request( - 'PUT', '/v1/kv/%s' % (quote(key),), json_data=value) + 'PUT', '/v1/kv/%s' % (key,), json_data=value) def get_kv_keys(self, keys_path, separator=None): """ @@ -254,11 +256,10 @@ def get_kv_keys(self, keys_path, separator=None): getting all the keys non-recursively for a path. For more information see the Consul API documentation. """ - params = {'keys': ''} + query = {'keys': None} if separator: - params['separator'] = separator - return self.get_json( - '/v1/kv/%s?%s' % (quote(keys_path), urlencode(params),)) + query['separator'] = separator + return self.get_json('/v1/kv/%s' % (keys_path,), query) def delete_kv_keys(self, key, recurse=False): """ @@ -269,8 +270,8 @@ def delete_kv_keys(self, key, recurse=False): :param: recurse: Whether or not to recursively delete all subpaths of the key. """ - query = '?recurse' if recurse else '' - return self.request('DELETE', '/v1/kv/%s%s' % (quote(key), query,)) + query = {'recurse': None} if recurse else None + return self.request('DELETE', '/v1/kv/%s' % (key,), query) def get_catalog_nodes(self): """ diff --git a/consular/tests/test_clients.py b/consular/tests/test_clients.py index b15e839..6dfd8e0 100644 --- a/consular/tests/test_clients.py +++ b/consular/tests/test_clients.py @@ -7,6 +7,8 @@ from txfake import FakeHttpServer from txfake.fake_connection import wait0 +from uritools import urisplit + from consular.clients import ( ConsulClient, HTTPError, JsonClient, MarathonClient) @@ -38,6 +40,14 @@ def write_json_response(self, request, json_data, response_code=200, def uri(self, path): return '%s%s' % (self.client.endpoint, path,) + def parse_query(self, uri): + """ + When Twisted parses "args" from the URI, it leaves out query parameters + that have no value. In those cases we rather use uritools to parse the + query parameters. + """ + return urisplit(uri).getquerydict() + class JsonClientTest(JsonClientTestBase): @@ -604,8 +614,8 @@ def test_get_kv_keys(self): request = yield self.requests.get() self.assertEqual(request.method, 'GET') self.assertEqual(request.path, self.uri('/v1/kv/foo')) - self.assertEqual(request.args, { - 'keys': [''] + self.assertEqual(self.parse_query(request.uri), { + 'keys': [None] }) keys = [ @@ -631,8 +641,8 @@ def test_get_kv_keys_separator(self): request = yield self.requests.get() self.assertEqual(request.method, 'GET') self.assertEqual(request.path, self.uri('/v1/kv/foo')) - self.assertEqual(request.args, { - 'keys': [''], + self.assertEqual(self.parse_query(request.uri), { + 'keys': [None], 'separator': ['/'] }) @@ -674,11 +684,10 @@ def test_delete_kv_keys_recursive(self): request = yield self.requests.get() self.assertEqual(request.method, 'DELETE') - self.assertEqual(request.uri, self.uri('/v1/kv/foo?recurse')) - # request.args == {} :'( - # self.assertEqual(request.args, { - # 'recurse': [] - # }) + self.assertEqual(request.path, self.uri('/v1/kv/foo')) + self.assertEqual(self.parse_query(request.uri), { + 'recurse': [None] + }) request.setResponseCode(200) request.finish() diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 2270c70..931e45a 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -1,5 +1,4 @@ import json -from urllib import urlencode from twisted.trial.unittest import TestCase from twisted.web.server import Site @@ -222,7 +221,7 @@ def test_TASK_RUNNING(self): consul_kv_request = yield self.requests.get() self.assertEqual(consul_kv_request['method'], 'GET') self.assertEqual(consul_kv_request['url'], - 'http://localhost:8500/v1/kv/consular/my-app?keys=') + 'http://localhost:8500/v1/kv/consular/my-app?keys') consul_kv_request['deferred'].callback( FakeResponse(200, [], json.dumps([]))) @@ -331,7 +330,7 @@ def test_TASK_RUNNING_no_ports(self): consul_kv_request = yield self.requests.get() self.assertEqual(consul_kv_request['method'], 'GET') self.assertEqual(consul_kv_request['url'], - 'http://localhost:8500/v1/kv/consular/my-app?keys=') + 'http://localhost:8500/v1/kv/consular/my-app?keys') consul_kv_request['deferred'].callback( FakeResponse(200, [], json.dumps([]))) @@ -394,7 +393,7 @@ def test_TASK_RUNNING_multiple_ports(self): consul_kv_request = yield self.requests.get() self.assertEqual(consul_kv_request['method'], 'GET') self.assertEqual(consul_kv_request['url'], - 'http://localhost:8500/v1/kv/consular/my-app?keys=') + 'http://localhost:8500/v1/kv/consular/my-app?keys') consul_kv_request['deferred'].callback( FakeResponse(200, [], json.dumps([]))) @@ -513,10 +512,8 @@ def test_register_with_marathon(self): create_callback_request = yield self.requests.get() self.assertEqual( create_callback_request['url'], - 'http://localhost:8080/v2/eventSubscriptions?%s' % (urlencode({ - 'callbackUrl': ('http://localhost:7000/' - 'events?registration=the-uuid') - }),)) + 'http://localhost:8080/v2/eventSubscriptions?' + 'callbackUrl=http://localhost:7000/events?registration=the-uuid') self.assertEqual(create_callback_request['method'], 'POST') create_callback_request['deferred'].callback(FakeResponse(200, [])) @@ -754,7 +751,7 @@ def test_sync_app_labels(self): consul_request = yield self.requests.get() self.assertEqual(consul_request['method'], 'GET') self.assertEqual(consul_request['url'], - 'http://localhost:8500/v1/kv/consular/my-app?keys=') + 'http://localhost:8500/v1/kv/consular/my-app?keys') consul_request['deferred'].callback( FakeResponse(200, [], json.dumps([]))) @@ -783,7 +780,7 @@ def test_sync_app_labels_cleanup(self): get_request = yield self.requests.get() self.assertEqual(get_request['method'], 'GET') self.assertEqual(get_request['url'], - 'http://localhost:8500/v1/kv/consular/my-app?keys=') + 'http://localhost:8500/v1/kv/consular/my-app?keys') consul_labels = [ 'consular/my-app/foo', 'consular/my-app/oldfoo', @@ -831,7 +828,7 @@ def test_sync_app_labels_cleanup_not_found(self): get_request = yield self.requests.get() self.assertEqual(get_request['method'], 'GET') self.assertEqual(get_request['url'], - 'http://localhost:8500/v1/kv/consular/my-app?keys=') + 'http://localhost:8500/v1/kv/consular/my-app?keys') get_request['deferred'].callback(FakeResponse(404, [], None)) yield d @@ -858,7 +855,7 @@ def test_sync_app_labels_cleanup_forbidden(self): get_request = yield self.requests.get() self.assertEqual(get_request['method'], 'GET') self.assertEqual(get_request['url'], - 'http://localhost:8500/v1/kv/consular/my-app?keys=') + 'http://localhost:8500/v1/kv/consular/my-app?keys') get_request['deferred'].callback(FakeResponse(403, [], None)) # Error is raised into a DeferredList, must get actual error @@ -866,7 +863,7 @@ def test_sync_app_labels_cleanup_forbidden(self): self.assertEqual( failure.getErrorMessage(), '403 Client Error for url: ' - 'http://localhost:8500/v1/kv/consular/my-app?keys=') + 'http://localhost:8500/v1/kv/consular/my-app?keys') @inlineCallbacks def test_sync_app(self): @@ -882,7 +879,7 @@ def test_sync_app(self): self.assertEqual(consul_request['method'], 'GET') self.assertEqual( consul_request['url'], - 'http://localhost:8500/v1/kv/consular/my-app?keys=') + 'http://localhost:8500/v1/kv/consular/my-app?keys') consul_request['deferred'].callback( FakeResponse(200, [], json.dumps([]))) @@ -1253,7 +1250,7 @@ def test_purge_dead_app_labels(self): self.assertEqual(consul_request['method'], 'GET') self.assertEqual( consul_request['url'], - 'http://localhost:8500/v1/kv/consular/?keys=&separator=%2F') + 'http://localhost:8500/v1/kv/consular/?keys&separator=/') # Return one existing app and one non-existing app consul_request['deferred'].callback( FakeResponse(200, [], json.dumps([ @@ -1286,7 +1283,7 @@ def test_purge_dead_app_labels_not_found(self): self.assertEqual(consul_request['method'], 'GET') self.assertEqual( consul_request['url'], - 'http://localhost:8500/v1/kv/consular/?keys=&separator=%2F') + 'http://localhost:8500/v1/kv/consular/?keys&separator=/') # Return a 404 error consul_request['deferred'].callback(FakeResponse(404, [], None)) @@ -1306,7 +1303,7 @@ def test_purge_dead_app_labels_forbidden(self): self.assertEqual(consul_request['method'], 'GET') self.assertEqual( consul_request['url'], - 'http://localhost:8500/v1/kv/consular/?keys=&separator=%2F') + 'http://localhost:8500/v1/kv/consular/?keys&separator=/') # Return a 403 error consul_request['deferred'].callback(FakeResponse(403, [], None)) @@ -1314,7 +1311,7 @@ def test_purge_dead_app_labels_forbidden(self): self.assertEqual( failure.getErrorMessage(), '403 Client Error for url: ' - 'http://localhost:8500/v1/kv/consular/?keys=&separator=%2F') + 'http://localhost:8500/v1/kv/consular/?keys&separator=/') @inlineCallbacks def test_fallback_to_main_consul(self): diff --git a/setup.py b/setup.py index bb91d83..b3cf714 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ 'Klein', 'treq', 'Twisted', + 'uritools>=1.0.0' ], entry_points={ 'console_scripts': ['consular = consular.cli:main'], From 9a6a9f12958ec648dcc0442e2a0ec3eb540e6170 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Thu, 18 Feb 2016 14:43:48 +0200 Subject: [PATCH 096/111] Build Consul Agent endpoints from address and known endpoint --- consular/clients.py | 29 ++++++++++++++++++++--------- consular/main.py | 16 ++++------------ consular/tests/test_clients.py | 13 +++++-------- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/consular/clients.py b/consular/clients.py index b0564f1..05bf294 100644 --- a/consular/clients.py +++ b/consular/clients.py @@ -22,7 +22,7 @@ def __init__(self, endpoint): """ Create a client with the specified default endpoint. """ - self.endpoint = endpoint + self.endpoint = urisplit(endpoint) self.pool = client.HTTPConnectionPool(self.clock, persistent=False) def requester(self, *args, **kwargs): @@ -61,7 +61,8 @@ def request(self, method, path, query=None, endpoint=None, json_data=None, Any other parameters that will be passed to `treq.request`, for example headers or parameters. """ - scheme, authority = urisplit(endpoint or self.endpoint)[:2] + scheme, authority = (urisplit(endpoint)[:2] if endpoint is not None + else self.endpoint[:2]) url = uricompose(scheme, authority, path, query) data = json.dumps(json_data) if json_data else None @@ -205,13 +206,21 @@ def __init__(self, endpoint, enable_fallback=False): on an agent that cannot be reached. """ super(ConsulClient, self).__init__(endpoint) - self.endpoint = endpoint self.enable_fallback = enable_fallback - def register_agent_service(self, agent_endpoint, registration): + def _get_agent_endpoint(self, agent_address): """ - Register a Consul service at the given agent endpoint. + Use the default endpoint to construct the agent endpoint from an + address, i.e. use the same scheme and port but swap in the address. """ + return uricompose(scheme=self.endpoint.scheme, host=agent_address, + port=self.endpoint.port) + + def register_agent_service(self, agent_address, registration): + """ + Register a Consul service at the given agent address. + """ + agent_endpoint = self._get_agent_endpoint(agent_address) d = self.request('PUT', '/v1/agent/service/register', endpoint=agent_endpoint, json_data=registration) @@ -231,10 +240,11 @@ def _register_agent_service_fallback(self, failure, registration): 'PUT', '/v1/agent/service/register', json_data=registration, timeout=self.fallback_timeout) - def deregister_agent_service(self, agent_endpoint, service_id): + def deregister_agent_service(self, agent_address, service_id): """ - Deregister a Consul service at the given agent endpoint. + Deregister a Consul service at the given agent address. """ + agent_endpoint = self._get_agent_endpoint(agent_address) return self.request('PUT', '/v1/agent/service/deregister/%s' % ( service_id,), endpoint=agent_endpoint) @@ -279,8 +289,9 @@ def get_catalog_nodes(self): """ return self.get_json('/v1/catalog/nodes') - def get_agent_services(self, agent_endpoint): + def get_agent_services(self, agent_address): """ - Get the list of running services for the given agent endpoint. + Get the list of running services for the given agent address. """ + agent_endpoint = self._get_agent_endpoint(agent_address) return self.get_json('/v1/agent/services', endpoint=agent_endpoint) diff --git a/consular/main.py b/consular/main.py index 1429328..16de8dd 100644 --- a/consular/main.py +++ b/consular/main.py @@ -20,10 +20,6 @@ def get_app_name(app_id): return app_id.lstrip('/').replace('/', '-') -def get_agent_endpoint(host): - return 'http://%s:8500' % (host,) - - @inlineCallbacks def handle_not_found_error(f, *args, **kwargs): """ @@ -334,14 +330,12 @@ def register_task_service(self, app_id, task_id, host, ports): 'only supports a single port. Only the lowest port (%s) ' 'will be used.' % (len(ports), app_id, port,)) - agent_endpoint = get_agent_endpoint(host) log.msg('Registering %s at %s with %s at %s:%s.' % ( - app_id, agent_endpoint, task_id, host, port)) + app_id, host, task_id, host, port)) registration = self._create_service_registration(app_id, task_id, host, port) - return self.consul_client.register_agent_service( - agent_endpoint, registration) + return self.consul_client.register_agent_service(host, registration) def deregister_task_service(self, task_id, host): """ @@ -352,8 +346,7 @@ def deregister_task_service(self, task_id, host): :param str host: The host address of the machine the task is running on. """ - return self.deregister_consul_service( - get_agent_endpoint(host), task_id) + return self.deregister_consul_service(host, task_id) def deregister_consul_service(self, agent_endpoint, service_id): """ @@ -590,8 +583,7 @@ def _consul_key_to_marathon_app_name(self, consul_key): def purge_dead_services(self): nodes = yield self.consul_client.get_catalog_nodes() for node in nodes: - self.purge_dead_agent_services( - get_agent_endpoint(node['Address'])) + self.purge_dead_agent_services(node['Address']) @inlineCallbacks def purge_dead_agent_services(self, agent_endpoint): diff --git a/consular/tests/test_clients.py b/consular/tests/test_clients.py index 6dfd8e0..cdc6cc8 100644 --- a/consular/tests/test_clients.py +++ b/consular/tests/test_clients.py @@ -38,7 +38,7 @@ def write_json_response(self, request, json_data, response_code=200, request.finish() def uri(self, path): - return '%s%s' % (self.client.endpoint, path,) + return '%s%s' % (self.client.endpoint.geturi(), path,) def parse_query(self, uri): """ @@ -501,8 +501,7 @@ def test_register_agent_service(self): 'TTL': '15s' } } - d = self.client.register_agent_service('http://foo.example.com:8500', - registration) + d = self.client.register_agent_service('foo.example.com', registration) request = yield self.requests.get() self.assertEqual(request.method, 'PUT') @@ -540,8 +539,7 @@ def test_register_agent_service_fallback(self): 'TTL': '15s' } } - d = self.client.register_agent_service('http://foo.example.com:8500', - registration) + d = self.client.register_agent_service('foo.example.com', registration) request = yield self.requests.get() # Fail the request @@ -566,8 +564,7 @@ def test_deregister_agent_service(self): When a service is deregistered, a PUT request is made to the correct address. """ - d = self.client.deregister_agent_service('http://foo.example.com:8500', - 'redis1') + d = self.client.deregister_agent_service('foo.example.com', 'redis1') request = yield self.requests.get() self.assertEqual(request.method, 'PUT') @@ -729,7 +726,7 @@ def test_get_agent_services(self): When we get the list of services from an agent, a request is made to the correct address and a list of services is returned. """ - d = self.client.get_agent_services('http://foo.example.com:8500') + d = self.client.get_agent_services('foo.example.com') request = yield self.requests.get() self.assertEqual(request.method, 'GET') From 8041af3aa17487a6753aa77d43895fa5ff956265 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Thu, 18 Feb 2016 14:58:50 +0200 Subject: [PATCH 097/111] Don't split inline if block across 2 lines --- consular/clients.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/consular/clients.py b/consular/clients.py index 05bf294..73d4163 100644 --- a/consular/clients.py +++ b/consular/clients.py @@ -61,8 +61,10 @@ def request(self, method, path, query=None, endpoint=None, json_data=None, Any other parameters that will be passed to `treq.request`, for example headers or parameters. """ - scheme, authority = (urisplit(endpoint)[:2] if endpoint is not None - else self.endpoint[:2]) + if endpoint is not None: + scheme, authority = urisplit(endpoint)[:2] + else: + scheme, authority = self.endpoint[:2] url = uricompose(scheme, authority, path, query) data = json.dumps(json_data) if json_data else None From 58bc66b8bcb0c357542ff1560b67e8d3c7cbe0ad Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 16 Feb 2016 14:35:25 +0200 Subject: [PATCH 098/111] Move install requirements to setup.py and clean up Travis file --- .travis.yml | 7 +++---- requirements.txt | 6 ++---- setup.py | 11 ++++++----- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 544cb27..d491cae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,13 +6,12 @@ cache: directories: - $HOME/.cache/pip install: - - pip install "cryptography<=1.0" # NOTE: because pypy<2.5 on Travis - - pip install twine - - pip install coveralls - pip install --upgrade pip - - pip install flake8 + - pip install "cryptography<=1.0" # NOTE: because pypy<2.5 on Travis - pip install -r requirements-dev.txt - pip install -e . + - pip install coveralls + - pip install twine script: - flake8 consular - py.test consular --cov consular diff --git a/requirements.txt b/requirements.txt index 903109c..0b5bd7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,2 @@ -Twisted -Klein -treq -click +# Our dependencies are all specified in setup.py. +-e . diff --git a/setup.py b/setup.py index 43a96fb..bb91d83 100644 --- a/setup.py +++ b/setup.py @@ -6,9 +6,6 @@ with open(os.path.join(here, 'README.rst')) as f: README = f.read() -with open(os.path.join(here, 'requirements.txt')) as f: - requires = filter(None, f.readlines()) - with open(os.path.join(here, 'VERSION')) as f: version = f.read().strip() @@ -29,8 +26,12 @@ packages=find_packages(exclude=['docs']), include_package_data=True, zip_safe=False, - install_requires=requires, - tests_require=requires, + install_requires=[ + 'click', + 'Klein', + 'treq', + 'Twisted', + ], entry_points={ 'console_scripts': ['consular = consular.cli:main'], }) From 93ad59def349b971a5f3df6cf48a7d658489cb3c Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 16 Feb 2016 14:45:14 +0200 Subject: [PATCH 099/111] Use newer PyPy --- .travis.yml | 21 +++++++++++++++------ setup-pypy-travis.sh | 16 ++++++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 setup-pypy-travis.sh diff --git a/.travis.yml b/.travis.yml index d491cae..4f635b4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,22 +1,31 @@ language: python python: - "2.7" - - "pypy" +matrix: + include: + - python: "2.7" + - python: "pypy" + env: PYPY_VERSION="4.0.1" NO_COVERAGE=1 cache: directories: - $HOME/.cache/pip + - $HOME/downloads + +before_install: + # If necessary, set up an appropriate version of pypy. + - if [ ! -z "$PYPY_VERSION" ]; then source setup-pypy-travis.sh; fi + - if [ ! -z "$PYPY_VERSION" ]; then python --version 2>&1 | fgrep "PyPy $PYPY_VERSION"; fi install: - pip install --upgrade pip - - pip install "cryptography<=1.0" # NOTE: because pypy<2.5 on Travis - - pip install -r requirements-dev.txt - - pip install -e . + - pip install -r requirements.txt - pip install coveralls - pip install twine script: - flake8 consular - - py.test consular --cov consular + - if [ -z "$NO_COVERAGE" ]; then COVERAGE_OPT="--cov consular"; else COVERAGE_OPT=""; fi + - py.test consular $COVERAGE_OPT after_success: - - coveralls + - if [ -z "$NO_COVERAGE" ]; then coveralls; fi deploy: provider: pypi user: smn diff --git a/setup-pypy-travis.sh b/setup-pypy-travis.sh new file mode 100644 index 0000000..05caa67 --- /dev/null +++ b/setup-pypy-travis.sh @@ -0,0 +1,16 @@ +# NOTE: This script needs to be sourced so it can modify the environment. + +# Get out of the virtualenv we're in. +deactivate + +# Install pyenv. +curl -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash +export PATH="$HOME/.pyenv/bin:$PATH" +eval "$(pyenv init -)" +eval "$(pyenv virtualenv-init -)" + +# Install pypy and make a virtualenv for it. +pyenv install -s pypy-$PYPY_VERSION +pyenv global pypy-$PYPY_VERSION +virtualenv -p $(which python) ~/env-pypy-$PYPY_VERSION +source ~/env-pypy-$PYPY_VERSION/bin/activate From 28b6a768a17f9273bebf04b173c41cf5fac05129 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 16 Feb 2016 14:48:29 +0200 Subject: [PATCH 100/111] Travis: Try fix requirements installation --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4f635b4..19f8295 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,8 @@ before_install: - if [ ! -z "$PYPY_VERSION" ]; then python --version 2>&1 | fgrep "PyPy $PYPY_VERSION"; fi install: - pip install --upgrade pip - - pip install -r requirements.txt + - pip install -e . + - pip install -r requirements-dev.txt - pip install coveralls - pip install twine script: From 8e09cad99cd4f578f544a99f2cf49b0a7e16fad7 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 16 Feb 2016 14:52:06 +0200 Subject: [PATCH 101/111] Travis: remove unnecesary python --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 19f8295..85b552a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ python: - "2.7" matrix: include: - - python: "2.7" - python: "pypy" env: PYPY_VERSION="4.0.1" NO_COVERAGE=1 cache: From 03388542fdb0754f7ec8773461c1543fb7f5e3aa Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 16 Feb 2016 12:15:07 +0200 Subject: [PATCH 102/111] Fixes for new flake8 checks --- consular/clients.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/consular/clients.py b/consular/clients.py index 7cbf17e..0c3a509 100644 --- a/consular/clients.py +++ b/consular/clients.py @@ -1,5 +1,6 @@ from urllib import quote, urlencode import json +import treq from twisted.internet import reactor from twisted.python import log @@ -9,14 +10,11 @@ # Twisted's default HTTP11 client factory is way too verbose client._HTTP11ClientFactory.noisy = False -import treq - class JsonClient(object): debug = False clock = reactor timeout = 5 - requester = lambda self, *a, **kw: treq.request(*a, **kw) def __init__(self, endpoint): """ @@ -25,6 +23,9 @@ def __init__(self, endpoint): self.endpoint = endpoint self.pool = client.HTTPConnectionPool(self.clock, persistent=False) + def requester(self, *args, **kwargs): + return treq.request(*args, **kwargs) + def _log_http_response(self, response, method, path, data): log.msg('%s %s with %s returned: %s' % ( method, path, data, response.code)) From 9e1f9b6055ece74f3fb92c91beb9c1421d4977de Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 10 May 2016 17:26:54 +0200 Subject: [PATCH 103/111] Check for 404 during task syncing --- consular/main.py | 10 +++++++++- consular/tests/test_main.py | 25 +++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/consular/main.py b/consular/main.py index 1429328..3f4ef48 100644 --- a/consular/main.py +++ b/consular/main.py @@ -529,7 +529,15 @@ def _consul_key_to_marathon_label_key(self, consul_key): @inlineCallbacks def sync_app_tasks(self, app): - tasks = yield self.marathon_client.get_app_tasks(app['id']) + tasks = yield handle_not_found_error( + self.marathon_client.get_app_tasks, app['id']) + if tasks is None: + # Certain versions of Marathon may return 404 when an app has no + # tasks. Other versions return an empty list. + # https://github.com/mesosphere/marathon/issues/3881 + log.msg('No tasks found in Marathon for app ID "%s"' % app['id']) + return + for task in tasks: yield self.register_task_service( app['id'], task['id'], task['host'], task['ports']) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 2270c70..3b096bb 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -736,6 +736,31 @@ def test_sync_app_tasks_multiple_ports(self): FakeResponse(200, [], json.dumps({}))) yield d + @inlineCallbacks + def test_sync_app_tasks_not_found(self): + """ + When syncing an app with a task, and Marathon has no tasks for the app, + Consular should handle a 404 response from Marathon gracefully. + """ + d = self.consular.sync_app_tasks({'id': '/my-app'}) + + # First Consular fetches the tasks for the app + marathon_request = yield self.requests.get() + self.assertEqual(marathon_request['method'], 'GET') + self.assertEqual( + marathon_request['url'], + 'http://localhost:8080/v2/apps/my-app/tasks') + + # Respond with a 404 + marathon_request['deferred'].callback( + FakeResponse(404, [], json.dumps( + {"message": "App '/my-app' does not exist"})) + ) + + # Nothing much should happen -- there are no tasks + + yield d + @inlineCallbacks def test_sync_app_labels(self): app = { From eb46313864aea50894b43219357cf76c1a0c5030 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 10 May 2016 17:55:29 +0200 Subject: [PATCH 104/111] Bump version to 1.2.1 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 26aaba0..6085e94 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.0 +1.2.1 From 609c2c66d75927146f7e89bb67b3b189b3692e88 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Tue, 23 Aug 2016 17:12:25 +0200 Subject: [PATCH 105/111] failing test --- consular/tests/test_main.py | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 931e45a..74e0a05 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -644,6 +644,60 @@ def test_sync_app_tasks_grouped(self): FakeResponse(200, [], json.dumps({}))) yield d + @inlineCallbacks + def test_sync_app_tasks_task_lost(self): + """ + When syncing an app with a task, Consul is updated with a service entry + for the task. + """ + d = self.consular.sync_app_tasks({'id': '/my-app'}) + + # First Consular fetches the tasks for the app + marathon_request = yield self.requests.get() + self.assertEqual(marathon_request['method'], 'GET') + self.assertEqual( + marathon_request['url'], + 'http://localhost:8080/v2/apps/my-app/tasks') + + # Respond with one task + marathon_request['deferred'].callback( + FakeResponse(200, [], json.dumps({ + 'tasks': [ + { + 'id': 'my-task-1', + 'host': '0.0.0.0', + 'ports': [1234], + 'state': 'TASK_LOST', + }, + { + 'id': 'my-task-2', + 'host': '0.0.0.0', + 'ports': [5678], + 'state': 'TASK_RUNNING', + } + ]})) + ) + + # Consular should register the task in Consul + consul_request = yield self.requests.get() + self.assertEqual( + consul_request['url'], + 'http://0.0.0.0:8500/v1/agent/service/register') + self.assertEqual(json.loads(consul_request['data']), { + 'Name': 'my-app', + 'ID': 'my-task-2', + 'Address': '0.0.0.0', + 'Port': 5678, + 'Tags': [ + 'consular-reg-id=test', + 'consular-app-id=/my-app', + ], + }) + self.assertEqual(consul_request['method'], 'PUT') + consul_request['deferred'].callback( + FakeResponse(200, [], json.dumps({}))) + yield d + @inlineCallbacks def test_sync_app_tasks_no_ports(self): """ From 881e0f128602c9679f97428a5857d5f01b2531aa Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Tue, 23 Aug 2016 17:17:53 +0200 Subject: [PATCH 106/111] fix tests --- consular/main.py | 5 +++-- consular/tests/test_main.py | 29 +++++++++++++++++++++-------- setup.cfg | 2 +- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/consular/main.py b/consular/main.py index 16de8dd..d65a21f 100644 --- a/consular/main.py +++ b/consular/main.py @@ -524,8 +524,9 @@ def _consul_key_to_marathon_label_key(self, consul_key): def sync_app_tasks(self, app): tasks = yield self.marathon_client.get_app_tasks(app['id']) for task in tasks: - yield self.register_task_service( - app['id'], task['id'], task['host'], task['ports']) + if task['state'] == 'TASK_RUNNING': + yield self.register_task_service( + app['id'], task['id'], task['host'], task['ports']) @inlineCallbacks def purge_dead_app_labels(self, apps): diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 74e0a05..6a60f01 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -79,8 +79,7 @@ def request(self, method, path, data=None): return treq.request( method, 'http://localhost:%s%s' % ( self.listener_port, - path - ), + path), data=(json.dumps(data) if data is not None else None), pool=self.pool) @@ -576,9 +575,12 @@ def test_sync_app_tasks(self): # Respond with one task marathon_request['deferred'].callback( FakeResponse(200, [], json.dumps({ - 'tasks': [ - {'id': 'my-task-id', 'host': '0.0.0.0', 'ports': [1234]} - ]})) + 'tasks': [{ + 'id': 'my-task-id', + 'host': '0.0.0.0', + 'ports': [1234], + 'state': 'TASK_RUNNING', + }]})) ) # Consular should register the task in Consul @@ -620,7 +622,12 @@ def test_sync_app_tasks_grouped(self): marathon_request['deferred'].callback( FakeResponse(200, [], json.dumps({ 'tasks': [ - {'id': 'my-task-id', 'host': '0.0.0.0', 'ports': [1234]} + { + 'id': 'my-task-id', + 'host': '0.0.0.0', + 'ports': [1234], + 'state': 'TASK_RUNNING', + } ]})) ) @@ -717,7 +724,12 @@ def test_sync_app_tasks_no_ports(self): marathon_request['deferred'].callback( FakeResponse(200, [], json.dumps({ 'tasks': [ - {'id': 'my-task-id', 'host': '0.0.0.0', 'ports': []} + { + 'id': 'my-task-id', + 'host': '0.0.0.0', + 'ports': [], + 'state': 'TASK_RUNNING', + } ]})) ) @@ -762,7 +774,8 @@ def test_sync_app_tasks_multiple_ports(self): { 'id': 'my-task-id', 'host': '0.0.0.0', - 'ports': [4567, 1234, 6789] + 'ports': [4567, 1234, 6789], + 'state': 'TASK_RUNNING', } ]})) ) diff --git a/setup.cfg b/setup.cfg index e18604d..28516f8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,2 @@ -[pytest] +[tool:pytest] addopts = --doctest-modules --verbose --ignore=ve/ From 614efb361bca8f150fb3f99b6aa8cd3dbb2f80fc Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Tue, 23 Aug 2016 17:35:57 +0200 Subject: [PATCH 107/111] add test for purging non TASK_RUNNING tasks --- consular/main.py | 4 +- consular/tests/test_main.py | 90 +++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/consular/main.py b/consular/main.py index d65a21f..1860f9d 100644 --- a/consular/main.py +++ b/consular/main.py @@ -631,6 +631,8 @@ def _filter_marathon_tasks(self, marathon_tasks, consul_service_ids): if not marathon_tasks: return consul_service_ids - task_id_set = set([task['id'] for task in marathon_tasks]) + task_id_set = set([task['id'] + for task in marathon_tasks + if task['state'] == 'TASK_RUNNING']) return [service_id for service_id in consul_service_ids if service_id not in task_id_set] diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 6a60f01..1a9d48b 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -1090,6 +1090,7 @@ def test_purge_dead_services(self): "id": "taskid2", "host": "machine-2", "ports": [8103], + "state": "TASK_RUNNING", "startedAt": "2015-07-14T14:54:31.934Z", "stagedAt": "2015-07-14T14:54:31.544Z", "version": "2015-07-14T13:07:32.095Z" @@ -1109,6 +1110,95 @@ def test_purge_dead_services(self): FakeResponse(200, [], json.dumps({}))) yield d + @inlineCallbacks + def test_purge_task_lost_services(self): + d = self.consular.purge_dead_services() + consul_request = yield self.requests.get() + consul_request['deferred'].callback( + FakeResponse(200, [], json.dumps([{ + 'Node': 'consul-node', + 'Address': '1.2.3.4', + }])) + ) + + agent_request = yield self.requests.get() + # Expecting a request to list of all services in Consul, + # returning 2 + self.assertEqual( + agent_request['url'], + 'http://1.2.3.4:8500/v1/agent/services') + self.assertEqual(agent_request['method'], 'GET') + agent_request['deferred'].callback( + FakeResponse(200, [], json.dumps({ + "testinggroup-someid1": { + "ID": "taskid1", + "Service": "testingapp", + "Tags": None, + "Address": "machine-1", + "Port": 8102, + "Tags": [ + "consular-reg-id=test", + "consular-app-id=/testinggroup/someid1", + ], + }, + "testinggroup-someid1": { + "ID": "taskid2", + "Service": "testingapp", + "Tags": None, + "Address": "machine-2", + "Port": 8103, + "Tags": [ + "consular-reg-id=test", + "consular-app-id=/testinggroup/someid1", + ], + } + })) + ) + + # Expecting a request for the tasks for a given app, returning + # 1 of which only has the TASK_RUNNING state. + testingapp_request = yield self.requests.get() + self.assertEqual(testingapp_request['url'], + 'http://localhost:8080/v2/apps/testinggroup/someid1/' + 'tasks') + self.assertEqual(testingapp_request['method'], 'GET') + testingapp_request['deferred'].callback( + FakeResponse(200, [], json.dumps({ + "tasks": [{ + "appId": "/testinggroup/someid1", + "id": "taskid1", + "host": "machine-1", + "ports": [8103], + "state": "TASK_RUNNING", + "startedAt": "2015-07-14T14:54:31.934Z", + "stagedAt": "2015-07-14T14:54:31.544Z", + "version": "2015-07-14T13:07:32.095Z" + }, { + "appId": "/testinggroup/someid1", + "id": "taskid2", + "host": "machine-2", + "ports": [8103], + "state": "TASK_LOST", + "startedAt": "2015-07-14T14:54:31.934Z", + "stagedAt": "2015-07-14T14:54:31.544Z", + "version": "2015-07-14T13:07:32.095Z" + }] + })) + ) + + # Expecting a service registering in Consul as a result for one + # of these services + deregister_request = yield self.requests.get() + self.assertEqual( + deregister_request['url'], + ('http://1.2.3.4:8500/v1/agent/service/deregister/' + 'testinggroup-someid1')) + self.assertEqual(deregister_request['method'], 'PUT') + deregister_request['deferred'].callback( + FakeResponse(200, [], json.dumps({}))) + yield d + + @inlineCallbacks def test_purge_old_services(self): """ From 4760c3c0f974f43291209982ba3c95494e55b949 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Tue, 23 Aug 2016 17:38:45 +0200 Subject: [PATCH 108/111] update docstrings and pin pytest --- consular/tests/test_main.py | 7 ++++++- requirements-dev.txt | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 1a9d48b..f3ebbc6 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -654,7 +654,8 @@ def test_sync_app_tasks_grouped(self): @inlineCallbacks def test_sync_app_tasks_task_lost(self): """ - When syncing an app with a task, Consul is updated with a service entry + When syncing an app with a task that has the TASK_LOST state, + Consul should not be updated with a service entry for the task. """ d = self.consular.sync_app_tasks({'id': '/my-app'}) @@ -1112,6 +1113,10 @@ def test_purge_dead_services(self): @inlineCallbacks def test_purge_task_lost_services(self): + """ + When a task has anything but the TASK_RUNNING state it should + be deregistered from Consul + """ d = self.consular.purge_dead_services() consul_request = yield self.requests.get() consul_request['deferred'].callback( diff --git a/requirements-dev.txt b/requirements-dev.txt index 8417d9e..78bdd69 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,4 @@ -pytest +pytest>=3.0.0 pytest-coverage pytest-xdist flake8 From 18148c91ba949f08b97c699bad7c91c86b648356 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Tue, 23 Aug 2016 17:39:57 +0200 Subject: [PATCH 109/111] more comment updates --- consular/tests/test_main.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index f3ebbc6..4f5b104 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -1160,8 +1160,8 @@ def test_purge_task_lost_services(self): })) ) - # Expecting a request for the tasks for a given app, returning - # 1 of which only has the TASK_RUNNING state. + # Expecting a request for the tasks for a given app, + # returning only 1 task with state `TASK_RUNNING` testingapp_request = yield self.requests.get() self.assertEqual(testingapp_request['url'], 'http://localhost:8080/v2/apps/testinggroup/someid1/' @@ -1203,7 +1203,6 @@ def test_purge_task_lost_services(self): FakeResponse(200, [], json.dumps({}))) yield d - @inlineCallbacks def test_purge_old_services(self): """ From c6f3953b360b6660b825f736636bdfaa5968a0d4 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Tue, 23 Aug 2016 18:13:10 +0200 Subject: [PATCH 110/111] add uritools dependency back in --- setup.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index bb91d83..f84670d 100644 --- a/setup.py +++ b/setup.py @@ -27,10 +27,11 @@ include_package_data=True, zip_safe=False, install_requires=[ - 'click', - 'Klein', - 'treq', - 'Twisted', + 'click', + 'Klein', + 'treq', + 'Twisted', + 'uritools>=1.0.0', ], entry_points={ 'console_scripts': ['consular = consular.cli:main'], From aead96cfab947e83613209539a7ec97787544b35 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Tue, 23 Aug 2016 19:21:44 +0200 Subject: [PATCH 111/111] version bump --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 6085e94..f0bb29e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.1 +1.3.0