diff --git a/.deploy.yaml b/.deploy.yaml new file mode 100644 index 0000000..3c2bcd8 --- /dev/null +++ b/.deploy.yaml @@ -0,0 +1,9 @@ +name: consular +user: ubuntu +buildscript: sideloader/deploy.sh +postinstall: sideloader/postinstall.sh +virtualenv_prefix: consular +pip: [] +dependencies: + - libssl-dev + - libffi-dev diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..70345d0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*.egg-info +*.pyc +.coverage +_trial_temp/ +/docs/_build diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..14f490a --- /dev/null +++ b/.travis.yml @@ -0,0 +1,26 @@ +language: python +python: + - "2.7" + - "pypy" +cache: + directories: + - $HOME/.cache/pip +install: + - pip install coveralls + - pip install --upgrade pip + - pip install flake8 + - pip install -r requirements-dev.txt + - pip install -e . +script: + - flake8 consular + - py.test consular --cov consular +after_success: + - coveralls +deploy: + provider: pypi + user: smn + password: + secure: "J2nhv8v9jd9uTtXVV6UR55XZCw2cqdik7AvXfHN+gueBNYat6p66Cwv0q1f686UErkuVexJzKUiEkW2fAdV45rVPlYQpZk2gF2nySWmDhvaAetUAI0VOTD80tqIC84kD3tz1MyPX+nNfDa5KD8hmOGM0jRfzibUb6hsgUzc3jxQ1H88mXiWXLwJVvKmntdMNlA+2XiW1YiRguFNHZSHWP0taiCVwp3/53PJyXztfuIO6/HsTi2uCd1zhyae60kx5WfeohanyvW7Fdeo3uNyaqO7je6pjKCZP1NGzMes8ul/YCQ3k+hL32PV/z+CINMjxiiwFH7mJnztevaLkdcXFehX31ir9nxhMUqhrK5lVVwuokbtZ+WUkBy5w3n67oHAIlfXPk9GPoB3wRukae3nfqQMn55ukLXb1PaEwPEbHzTQ4xDjk12b5wAsArWwTORliBuy1yuCpAl1D+1C9k8QlC1cMa8r2+pMaY4nST0WPAtRCfGwogcIqxj7lEfVA0RajCEea24BYmv67OWbNr3ANcoOyg8x9wfEaJfW3ptingPp01dgK7jpc1ze8Ch/iy0mXc/BDz1Tgq4wO02sFKj+KTiVjwNh5bHF9Rzn9n7CS5lBbAjMJCnnqaAFxuaXTCPPBDPnNKn24l3heP+brGnOqVlszvEslj539XItoEqtT4Ug=" + on: + tags: true + all_branches: true diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..a103a14 --- /dev/null +++ b/LICENSE @@ -0,0 +1,23 @@ +Copyright (c) 2014, Simon de Haan +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..87ab717 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,5 @@ +include VERSION +include LICENSE +include README.rst +include requirements.txt +recursive-include consular * diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..611c82f --- /dev/null +++ b/README.rst @@ -0,0 +1,33 @@ +Consular +======== + +Receive events from Marathon_, update Consul_ with the relevant information +about services & tasks. + +.. image:: https://travis-ci.org/universalcore/consular.svg?branch=develop + :target: https://travis-ci.org/universalcore/consular + :alt: Continuous Integration + +.. image:: https://coveralls.io/repos/universalcore/consular/badge.png?branch=develop + :target: https://coveralls.io/r/universalcore/consular?branch=develop + :alt: Code Coverage + +.. image:: https://readthedocs.org/projects/consular/badge/?version=latest + :target: https://consular.readthedocs.org + :alt: Consular Documentation + +.. image:: https://badge.fury.io/py/consular.svg + :target: https://pypi.python.org/pypi/consular + :alt: Pypi Package + +Usage +~~~~~ + +:: + + $ pip install consular + $ consular --help + + +.. _Marathon: http://mesosphere.github.io/marathon/ +.. _Consul: http://consul.io/ diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..7dea76e --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.0.1 diff --git a/consular/__init__.py b/consular/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/consular/cli.py b/consular/cli.py new file mode 100644 index 0000000..21dbaa9 --- /dev/null +++ b/consular/cli.py @@ -0,0 +1,79 @@ +import click +import sys + +from urllib import urlencode + + +@click.command() +@click.option('--scheme', default='http', + help='The scheme to use. (HTTP/HTTPS)') +@click.option('--host', default='localhost', + help='The host to bind to.') +@click.option('--port', default='7000', type=int, + help='The port to listen to.') +@click.option('--consul', default='http://localhost:8500', + help='The Consul HTTP API') +@click.option('--marathon', default='http://localhost:8080', + help='The Marathon HTTP API') +@click.option('--registration-id', + help=('Auto register for Marathon event callbacks with the ' + 'registration-id. Must be unique for each consular ' + 'process.'), type=str) +@click.option('--sync-interval', + help=('Automatically sync the apps in Marathon with what\'s ' + 'in Consul every _n_ seconds. Defaults to 0 (disabled).'), + type=int) +@click.option('--purge/--no-purge', + help=('Automatically purge dead services from Consul if they ' + 'are not known in Marathon ' + '(needs sync-interval enabled).'), + default=False) +@click.option('--logfile', + help='Where to log output to', + type=click.File('a'), + default=sys.stdout) +@click.option('--debug/--no-debug', + help='Log debug output or not', + default=False) +@click.option('--timeout', + help='HTTP API client timeout', + default=5, type=int) +@click.option('--fallback/--no-fallback', + help=('Fallback to the default Consul agent for service ' + 'registration if the host running the mesos tasks ' + 'is not running a consul agent. ' + 'ONLY USE IF YOU KNOW WHAT YOU ARE DOING.'), + default=False) +@click.option('--fallback-timeout', + help=('How long to wait until assuming there is no consul ' + 'agent running on a mesos-slave machine'), + default=2, type=int) +def main(scheme, host, port, + consul, marathon, registration_id, + sync_interval, purge, logfile, debug, timeout, + fallback, fallback_timeout): # pragma: no cover + from consular.main import Consular + from twisted.internet.task import LoopingCall + from twisted.internet import reactor + from twisted.python import log + + log.startLogging(logfile) + + consular = Consular(consul, marathon, fallback) + consular.debug = debug + consular.timeout = timeout + consular.fallback_timeout = fallback_timeout + if registration_id: + events_url = "%s://%s:%s/events?%s" % ( + scheme, host, port, + urlencode({ + 'registration': registration_id, + })) + consular.register_marathon_event_callback(events_url) + + if sync_interval > 0: + lc = LoopingCall(consular.sync_apps, purge) + lc.start(sync_interval, now=True) + + consular.run(host, port) + reactor.run() diff --git a/consular/main.py b/consular/main.py new file mode 100644 index 0000000..67b7a19 --- /dev/null +++ b/consular/main.py @@ -0,0 +1,309 @@ +import json + +from urllib import quote, urlencode +from twisted.internet import reactor +from twisted.web import client, server +# Twisted's fault HTTP11 client factory is way too verbose +client._HTTP11ClientFactory.noisy = False +from twisted.internet.defer import ( + succeed, inlineCallbacks, returnValue, gatherResults) +from twisted.python import log + + +import treq +from klein import Klein + + +def get_appid(app_id_string): + return app_id_string.rsplit('/', 1)[1] + + +def get_agent_endpoint(host): + return 'http://%s:8500' % (host,) + + +class ConsularSite(server.Site): + + debug = False + + def log(self, request): + if self.debug: + server.Site.log(self, request) + + +class Consular(object): + + app = Klein() + debug = False + clock = reactor + timeout = 5 + fallback_timeout = 2 + requester = lambda self, *a, **kw: treq.request(*a, **kw) + + def __init__(self, consul_endpoint, marathon_endpoint, enable_fallback): + self.consul_endpoint = consul_endpoint + self.marathon_endpoint = marathon_endpoint + self.pool = client.HTTPConnectionPool(self.clock, persistent=False) + self.enable_fallback = enable_fallback + self.event_dispatch = { + 'status_update_event': self.handle_status_update_event, + } + + def run(self, host, port): + site = ConsularSite(self.app.resource()) + site.debug = self.debug + self.clock.listenTCP(port, site, interface=host) + + def get_marathon_event_callbacks(self): + d = self.marathon_request('GET', '/v2/eventSubscriptions') + d.addErrback(log.err) + d.addCallback(lambda response: response.json()) + d.addCallback(self.get_marathon_event_callbacks_from_json) + return d + + def get_marathon_event_callbacks_from_json(self, json): + """ + Marathon may return a bad response when we get the existing event + callbacks. A common cause for this is that Marathon is not properly + configured. Raise an exception with information from Marathon if this + is the case, else return the callback URLs from the JSON response. + """ + if 'callbackUrls' not in json: + raise RuntimeError('Unable to get existing event callbacks from ' + + 'Marathon: %r' % (str(json),)) + + return json['callbackUrls'] + + def create_marathon_event_callback(self, url): + d = self.marathon_request( + 'POST', '/v2/eventSubscriptions?%s' % urlencode({ + 'callbackUrl': url, + })) + d.addErrback(log.err) + d.addCallback(lambda response: response.code == 200) + return d + + @inlineCallbacks + def register_marathon_event_callback(self, events_url): + existing_callbacks = yield self.get_marathon_event_callbacks() + already_registered = any( + [events_url == url for url in existing_callbacks]) + if already_registered: + log.msg('Consular event callback already registered.') + returnValue(True) + + registered = yield self.create_marathon_event_callback(events_url) + if registered: + log.msg('Consular event callback registered.') + else: + log.err('Consular event callback registration failed.') + returnValue(registered) + + def log_http_response(self, response, method, path, data): + log.msg('%s %s with %s returned: %s' % ( + method, path, data, response.code)) + return response + + def marathon_request(self, method, path, data=None): + return self._request( + method, '%s%s' % (self.marathon_endpoint, path), data) + + def consul_request(self, method, url, data=None): + return self._request(method, url, data, timeout=self.fallback_timeout) + + def _request(self, method, url, data, timeout=None): + d = self.requester( + method, + url.encode('utf-8'), + headers={ + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + data=(json.dumps(data) if data is not None else None), + pool=self.pool, + timeout=timeout or self.timeout) + if self.debug: + d.addCallback(self.log_http_response, method, url, data) + return d + + @app.route('/') + def index(self, request): + request.setHeader('Content-Type', 'application/json') + return json.dumps([]) + + @app.route('/events') + def events(self, request): + request.setHeader('Content-Type', 'application/json') + event = json.load(request.content) + handler = self.event_dispatch.get( + event.get('eventType'), self.handle_unknown_event) + return handler(request, event) + + def handle_status_update_event(self, request, event): + dispatch = { + 'TASK_STAGING': self.noop, + 'TASK_STARTING': self.noop, + 'TASK_RUNNING': self.update_task_running, + 'TASK_FINISHED': self.update_task_killed, + 'TASK_FAILED': self.update_task_killed, + 'TASK_KILLED': self.update_task_killed, + 'TASK_LOST': self.update_task_killed, + } + handler = dispatch.get(event['taskStatus']) + return handler(request, event) + + def noop(self, request, event): + return succeed(json.dumps({ + 'status': 'ok' + })) + + def update_task_running(self, request, event): + # NOTE: Marathon sends a list of ports, I don't know yet when & if + # there are multiple values in that list. + d = self.get_app(event['appId']) + d.addCallback(lambda app: self.sync_app(app)) + d.addCallback(lambda _: json.dumps({'status': 'ok'})) + return d + + def update_task_killed(self, request, event): + d = self.deregister_service( + get_agent_endpoint(event['host']), + get_appid(event['appId']), + event['taskId']) + d.addCallback(lambda _: json.dumps({'status': 'ok'})) + return d + + def handle_unknown_event(self, request, event): + event_type = event.get('eventType') + request.setHeader('Content-Type', 'application/json') + request.setResponseCode(400) # bad request + log.msg('Not handling event type: %s' % (event_type,)) + return json.dumps({ + 'error': 'Event type %s not supported.' % (event_type,) + }) + + def register_service(self, agent_endpoint, + app_id, service_id, address, port): + log.msg('Registering %s at %s with %s at %s:%s.' % ( + app_id, agent_endpoint, service_id, address, port)) + d = self.consul_request( + 'PUT', + '%s/v1/agent/service/register' % (agent_endpoint,), + { + 'Name': app_id, + 'ID': service_id, + 'Address': address, + 'Port': port, + }) + if self.enable_fallback: + d.addErrback( + self.register_service_fallback, app_id, service_id, + address, port) + return d + + def register_service_fallback(self, failure, + app_id, service_id, address, port): + log.msg('Falling back for %s at %s with %s at %s:%s.' % ( + app_id, self.consul_endpoint, service_id, address, port)) + return self.consul_request( + 'PUT', + '%s/v1/agent/service/register' % (self.consul_endpoint,), + { + 'Name': app_id, + 'ID': service_id, + 'Address': address, + 'Port': port, + }) + + def deregister_service(self, agent_endpoint, app_id, service_id): + log.msg('Deregistering %s at %s with %s' % ( + app_id, agent_endpoint, service_id,)) + return self.consul_request( + 'PUT', '%s/v1/agent/service/deregister/%s' % ( + agent_endpoint, service_id,)) + + def sync_apps(self, purge=False): + d = self.marathon_request('GET', '/v2/apps') + d.addCallback(lambda response: response.json()) + d.addCallback( + lambda data: gatherResults( + [self.sync_app(app) for app in data['apps']])) + if purge: + d.addCallback(lambda _: self.purge_dead_services()) + return d + + def get_app(self, app_id): + d = self.marathon_request('GET', '/v2/apps%s' % (app_id,)) + d.addCallback(lambda response: response.json()) + d.addCallback(lambda data: data['app']) + return d + + def sync_app(self, app): + return gatherResults([ + self.sync_app_labels(app), + self.sync_app_tasks(app), + ]) + + def sync_app_labels(self, app): + labels = app.get('labels', {}) + # NOTE: KV requests can go straight to the consul registry + # we're already connected to, they're not local to the agents. + return gatherResults([ + self.consul_request( + 'PUT', '%s/v1/kv/consular/%s/%s' % ( + self.consul_endpoint, + quote(get_appid(app['id'])), quote(key)), value) + for key, value in labels.items() + ]) + + def sync_app_tasks(self, app): + d = self.marathon_request('GET', '/v2/apps%(id)s/tasks' % app) + d.addCallback(lambda response: response.json()) + d.addCallback(lambda data: gatherResults( + self.sync_app_task(app, task) for task in data['tasks'])) + return d + + def sync_app_task(self, app, task): + return self.register_service( + get_agent_endpoint(task['host']), + get_appid(app['id']), task['id'], + task['host'], task['ports'][0]) + + def purge_dead_services(self): + d = self.consul_request( + 'GET', '%s/v1/catalog/nodes' % (self.consul_endpoint,)) + d.addCallback(lambda response: response.json()) + d.addCallback(lambda data: gatherResults([ + self.purge_dead_agent_services( + get_agent_endpoint(node['Address'])) for node in data + ])) + return d + + @inlineCallbacks + def purge_dead_agent_services(self, agent_endpoint): + response = yield self.consul_request( + 'GET', '%s/v1/agent/services' % (agent_endpoint,)) + data = yield response.json() + + # collect the task ids for the service name + services = {} + for service_id, service in data.items(): + services.setdefault(service['Service'], set([])).add(service_id) + + for app_id, task_ids in services.items(): + yield self.purge_service_if_dead(agent_endpoint, app_id, task_ids) + + @inlineCallbacks + def purge_service_if_dead(self, agent_endpoint, app_id, consul_task_ids): + response = yield self.marathon_request( + 'GET', '/v2/apps/%s/tasks' % (app_id,)) + data = yield response.json() + if 'tasks' not in data: + log.msg(('App %s does not look like a Marathon application, ' + 'skipping') % (str(app_id),)) + return + + marathon_task_ids = set([task['id'] for task in data['tasks']]) + tasks_to_be_purged = consul_task_ids - marathon_task_ids + for task_id in tasks_to_be_purged: + yield self.deregister_service(agent_endpoint, app_id, task_id) diff --git a/consular/tests/__init__.py b/consular/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py new file mode 100644 index 0000000..70e81b1 --- /dev/null +++ b/consular/tests/test_main.py @@ -0,0 +1,421 @@ +import json +from urllib import urlencode + +from twisted.trial.unittest import TestCase +from twisted.web.server import Site +from twisted.internet import reactor +from twisted.internet.defer import ( + inlineCallbacks, DeferredQueue, Deferred, succeed) +from twisted.web.client import HTTPConnectionPool +from twisted.python import log + +from consular.main import Consular + +import treq + + +class FakeResponse(object): + + def __init__(self, code, headers, content=None): + self.code = code + self.headers = headers + self._content = content + + def content(self): + return succeed(self._content) + + def json(self): + d = self.content() + d.addCallback(lambda content: json.loads(content)) + return d + + +class ConsularTest(TestCase): + + timeout = 1 + + def setUp(self): + self.consular = Consular( + 'http://localhost:8500', + 'http://localhost:8080', + False + ) + self.consular.debug = True + + # spin up a site so we can test it, pretty sure Klein has better + # ways of doing this but they're not documented anywhere. + self.site = Site(self.consular.app.resource()) + self.listener = reactor.listenTCP(0, self.site, interface='localhost') + self.listener_port = self.listener.getHost().port + self.addCleanup(self.listener.loseConnection) + + # cleanup stuff for treq's global http request pool + self.pool = HTTPConnectionPool(reactor, persistent=False) + self.addCleanup(self.pool.closeCachedConnections) + + # We use this to mock requests going to Consul & Marathon + self.requests = DeferredQueue() + + def mock_requests(method, url, headers, data, pool, timeout): + d = Deferred() + self.requests.put({ + 'method': method, + 'url': url, + 'data': data, + 'deferred': d, + }) + return d + + self.patch(self.consular, 'requester', mock_requests) + + def request(self, method, path, data=None): + return treq.request( + method, 'http://localhost:%s%s' % ( + self.listener_port, + path + ), + data=(json.dumps(data) if data is not None else None), + pool=self.pool) + + def tearDown(self): + pass + + @inlineCallbacks + def test_service(self): + response = yield self.request('GET', '/') + self.assertEqual(response.code, 200) + self.assertEqual((yield response.json()), []) + + @inlineCallbacks + def test_handle_unknown_event(self): + response = yield self.request('POST', '/events', {'eventType': 'Foo'}) + data = yield response.json() + self.assertEqual(data, { + 'error': 'Event type Foo not supported.' + }) + + @inlineCallbacks + def test_handle_unspecified_event(self): + response = yield self.request('POST', '/events', {}) + data = yield response.json() + self.assertEqual(data, { + 'error': 'Event type None not supported.' + }) + + @inlineCallbacks + def test_TASK_STAGING(self): + response = yield self.request('POST', '/events', { + "eventType": "status_update_event", + "timestamp": "2014-03-01T23:29:30.158Z", + "slaveId": "20140909-054127-177048842-5050-1494-0", + "taskId": "my-app_0-1396592784349", + "taskStatus": "TASK_STAGING", + "appId": "/my-app", + "host": "slave-1234.acme.org", + "ports": [31372], + "version": "2014-04-04T06:26:23.051Z" + }) + self.assertEqual((yield response.json()), { + 'status': 'ok' + }) + + @inlineCallbacks + def test_TASK_RUNNING(self): + d = self.request('POST', '/events', { + "eventType": "status_update_event", + "timestamp": "2014-03-01T23:29:30.158Z", + "slaveId": "20140909-054127-177048842-5050-1494-0", + "taskId": "my-app_0-1396592784349", + "taskStatus": "TASK_RUNNING", + "appId": "/my-app", + "host": "slave-1234.acme.org", + "ports": [31372], + "version": "2014-04-04T06:26:23.051Z" + }) + + # We should get the app info for the event + marathon_app_request = yield self.requests.get() + self.assertEqual(marathon_app_request['method'], 'GET') + self.assertEqual(marathon_app_request['url'], + 'http://localhost:8080/v2/apps/my-app') + marathon_app_request['deferred'].callback( + FakeResponse(200, [], json.dumps({ + 'app': { + 'id': '/my-app', + } + }))) + + # Then we collect the tasks for the app + marathon_tasks_request = yield self.requests.get() + self.assertEqual(marathon_tasks_request['method'], 'GET') + self.assertEqual(marathon_tasks_request['url'], + 'http://localhost:8080/v2/apps/my-app/tasks') + marathon_tasks_request['deferred'].callback( + FakeResponse(200, [], json.dumps({ + 'tasks': [{ + 'id': 'my-app_0-1396592784349', + 'host': 'slave-1234.acme.org', + 'ports': [31372], + }] + }))) + + request = yield self.requests.get() + self.assertEqual(request['method'], 'PUT') + self.assertEqual( + request['url'], + 'http://slave-1234.acme.org:8500/v1/agent/service/register') + self.assertEqual(request['data'], json.dumps({ + 'Name': 'my-app', + 'ID': 'my-app_0-1396592784349', + 'Address': 'slave-1234.acme.org', + 'Port': 31372, + })) + request['deferred'].callback( + FakeResponse(200, [], json.dumps({}))) + response = yield d + self.assertEqual((yield response.json()), { + 'status': 'ok' + }) + + @inlineCallbacks + def test_TASK_KILLED(self): + d = self.request('POST', '/events', { + "eventType": "status_update_event", + "timestamp": "2014-03-01T23:29:30.158Z", + "slaveId": "20140909-054127-177048842-5050-1494-0", + "taskId": "my-app_0-1396592784349", + "taskStatus": "TASK_KILLED", + "appId": "/my-app", + "host": "slave-1234.acme.org", + "ports": [31372], + "version": "2014-04-04T06:26:23.051Z" + }) + request = yield self.requests.get() + self.assertEqual(request['method'], 'PUT') + self.assertEqual( + request['url'], + ('http://slave-1234.acme.org:8500' + '/v1/agent/service/deregister/my-app_0-1396592784349')) + request['deferred'].callback( + FakeResponse(200, [], json.dumps({}))) + response = yield d + self.assertEqual((yield response.json()), { + 'status': 'ok' + }) + + @inlineCallbacks + def test_register_with_marathon(self): + d = self.consular.register_marathon_event_callback( + 'http://localhost:7000/events?registration=the-uuid') + d.addErrback(log.err) + list_callbacks_request = yield self.requests.get() + list_callbacks_request['deferred'].callback( + FakeResponse(200, [], json.dumps({'callbackUrls': []}))) + + create_callback_request = yield self.requests.get() + self.assertEqual( + create_callback_request['url'], + 'http://localhost:8080/v2/eventSubscriptions?%s' % (urlencode({ + 'callbackUrl': ('http://localhost:7000/' + 'events?registration=the-uuid') + }),)) + + self.assertEqual(create_callback_request['method'], 'POST') + create_callback_request['deferred'].callback(FakeResponse(200, [])) + response = yield d + self.assertEqual(response, True) + + @inlineCallbacks + def test_already_registered_with_marathon(self): + d = self.consular.register_marathon_event_callback( + 'http://localhost:7000/events?registration=the-uuid') + list_callbacks_request = yield self.requests.get() + list_callbacks_request['deferred'].callback( + FakeResponse(200, [], json.dumps({ + 'callbackUrls': [ + 'http://localhost:7000/events?registration=the-uuid' + ] + }))) + + response = yield d + self.assertEqual(response, True) + + @inlineCallbacks + def test_register_with_marathon_unexpected_response(self): + """ + When registering a Marathon event callback Consular checks if an event + callback already exists for itself. When we get the existing callbacks, + Consular should inform the user of any errors returned by Marathon. + """ + d = self.consular.register_marathon_event_callback( + 'http://localhost:7000/events?registration=the-uuid') + list_callbacks_request = yield self.requests.get() + list_callbacks_request['deferred'].callback( + FakeResponse(400, [], json.dumps({ + 'message': + 'http event callback system is not running on this Marathon ' + + 'instance. Please re-start this instance with ' + + '"--event_subscriber http_callback".'}))) + + failure = self.failureResultOf(d, RuntimeError) + self.assertEqual( + failure.getErrorMessage(), + 'Unable to get existing event callbacks from Marathon: ' + + '\'{u\\\'message\\\': u\\\'http event callback system is not ' + + 'running on this Marathon instance. Please re-start this ' + + 'instance with "--event_subscriber http_callback".\\\'}\'') + + @inlineCallbacks + def test_sync_app_task(self): + app = {'id': '/my-app'} + task = {'id': 'my-task-id', 'host': '0.0.0.0', 'ports': [1234]} + d = self.consular.sync_app_task(app, task) + consul_request = yield self.requests.get() + self.assertEqual( + consul_request['url'], + 'http://0.0.0.0:8500/v1/agent/service/register') + self.assertEqual(consul_request['data'], json.dumps({ + 'Name': 'my-app', + 'ID': 'my-task-id', + 'Address': '0.0.0.0', + 'Port': 1234, + })) + self.assertEqual(consul_request['method'], 'PUT') + consul_request['deferred'].callback( + FakeResponse(200, [], json.dumps({}))) + yield d + + @inlineCallbacks + def test_sync_app_labels(self): + app = { + 'id': '/my-app', + 'labels': {'foo': 'bar'} + } + d = self.consular.sync_app_labels(app) + consul_request = yield self.requests.get() + self.assertEqual(consul_request['method'], 'PUT') + self.assertEqual(consul_request['url'], + 'http://localhost:8500/v1/kv/consular/my-app/foo') + self.assertEqual(consul_request['data'], '"bar"') + consul_request['deferred'].callback( + FakeResponse(200, [], json.dumps({}))) + yield d + + @inlineCallbacks + def test_sync_app(self): + app = { + 'id': '/my-app', + } + d = self.consular.sync_app(app) + marathon_request = yield self.requests.get() + self.assertEqual( + marathon_request['url'], + 'http://localhost:8080/v2/apps/my-app/tasks') + self.assertEqual(marathon_request['method'], 'GET') + marathon_request['deferred'].callback( + FakeResponse(200, [], json.dumps({'tasks': []}))) + yield d + + @inlineCallbacks + def test_sync_apps(self): + d = self.consular.sync_apps(purge=False) + marathon_request = yield self.requests.get() + self.assertEqual(marathon_request['url'], + 'http://localhost:8080/v2/apps') + self.assertEqual(marathon_request['method'], 'GET') + marathon_request['deferred'].callback( + FakeResponse(200, [], json.dumps({'apps': []}))) + yield d + + @inlineCallbacks + def test_purge_dead_services(self): + d = self.consular.purge_dead_services() + consul_request = yield self.requests.get() + self.assertEqual( + consul_request['url'], + 'http://localhost:8500/v1/catalog/nodes') + consul_request['deferred'].callback( + FakeResponse(200, [], json.dumps([{ + 'Node': 'consul-node', + 'Address': '1.2.3.4', + }])) + ) + agent_request = yield self.requests.get() + # Expecting a request to list of all services in Consul, + # returning 2 + self.assertEqual( + agent_request['url'], + 'http://1.2.3.4:8500/v1/agent/services') + self.assertEqual(agent_request['method'], 'GET') + agent_request['deferred'].callback( + FakeResponse(200, [], json.dumps({ + "testingapp.someid1": { + "ID": "testingapp.someid1", + "Service": "testingapp", + "Tags": None, + "Address": "machine-1", + "Port": 8102 + }, + "testingapp.someid2": { + "ID": "testingapp.someid2", + "Service": "testingapp", + "Tags": None, + "Address": "machine-2", + "Port": 8103 + } + })) + ) + + # Expecting a request for the tasks for a given app, returning + # 1 less than Consul thinks exists. + testingapp_request = yield self.requests.get() + self.assertEqual(testingapp_request['url'], + 'http://localhost:8080/v2/apps/testingapp/tasks') + self.assertEqual(testingapp_request['method'], 'GET') + testingapp_request['deferred'].callback( + FakeResponse(200, [], json.dumps({ + "tasks": [{ + "appId": "/testingapp", + "id": "testingapp.someid2", + "host": "machine-2", + "ports": [8103], + "startedAt": "2015-07-14T14:54:31.934Z", + "stagedAt": "2015-07-14T14:54:31.544Z", + "version": "2015-07-14T13:07:32.095Z" + }] + })) + ) + + # Expecting a service registering in Consul as a result for one + # of these services + deregister_request = yield self.requests.get() + self.assertEqual( + deregister_request['url'], + ('http://1.2.3.4:8500/v1/agent/service/deregister/' + 'testingapp.someid1')) + self.assertEqual(deregister_request['method'], 'PUT') + deregister_request['deferred'].callback( + FakeResponse(200, [], json.dumps({}))) + yield d + + @inlineCallbacks + def test_fallback_to_main_consul(self): + self.consular.enable_fallback = True + self.consular.register_service( + 'http://foo:8500', 'app_id', 'service_id', 'foo', 1234) + request = yield self.requests.get() + self.assertEqual( + request['url'], + 'http://foo:8500/v1/agent/service/register') + request['deferred'].errback(Exception('Something terrible')) + + fallback_request = yield self.requests.get() + self.assertEqual( + fallback_request['url'], + 'http://localhost:8500/v1/agent/service/register') + self.assertEqual(fallback_request['data'], json.dumps({ + 'Name': 'app_id', + 'ID': 'service_id', + 'Address': 'foo', + 'Port': 1234, + })) diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..14ee9bb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,192 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + +clean: + rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Consular.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Consular.qhc" + +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/Consular" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Consular" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..4b22d4f --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,292 @@ +# -*- coding: utf-8 -*- +# +# Consular documentation build configuration file, created by +# sphinx-quickstart on Mon Jul 13 08:39:08 2015. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os +import shlex + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.viewcode', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Consular' +copyright = u'2015, Simon de Haan' +author = u'Simon de Haan' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = open('../VERSION').read() +# The full version, including alpha/beta/rc tags. +release = version + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' +#html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +#html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +#html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'Consulardoc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', + +# Latex figure (float) alignment +#'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'Consular.tex', u'Consular Documentation', + u'Simon de Haan', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'consular', u'Consular Documentation', + [author], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'Consular', u'Consular Documentation', + author, 'Consular', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'https://docs.python.org/': None} diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..29f298e --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,46 @@ +Consular +======== + +Receive events from Marathon_, update Consul_ with the relevant information +about services & tasks. + +.. image:: https://travis-ci.org/universalcore/consular.svg?branch=develop + :target: https://travis-ci.org/universalcore/consular + :alt: Continuous Integration + +.. image:: https://coveralls.io/repos/universalcore/consular/badge.png?branch=develop + :target: https://coveralls.io/r/universalcore/consular?branch=develop + :alt: Code Coverage + +.. image:: https://readthedocs.org/projects/consular/badge/?version=latest + :target: https://consular.readthedocs.org + :alt: Consular Documentation + +.. image:: https://badge.fury.io/py/consular.svg + :target: https://pypi.python.org/pypi/consular + :alt: Pypi Package + +Usage +~~~~~ + +:: + + $ pip install consular + $ consular --help + + +Installing for local dev +~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + $ git clone https://github.com/universalcore/consular.git + $ cd consular + $ virtualenv ve + $ source ve/bin/activate + (ve)$ pip install -e . + (ve)$ pip install -r requirements-dev.txt + + +.. _Marathon: http://mesosphere.github.io/marathon/ +.. _Consul: http://consul.io/ diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..7442659 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,263 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +set I18NSPHINXOPTS=%SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. xml to make Docutils-native XML files + echo. pseudoxml to make pseudoxml-XML files for display purposes + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + echo. coverage to run coverage check of the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + + +REM Check if sphinx-build is available and fallback to Python version if any +%SPHINXBUILD% 2> nul +if errorlevel 9009 goto sphinx_python +goto sphinx_ok + +:sphinx_python + +set SPHINXBUILD=python -m sphinx.__init__ +%SPHINXBUILD% 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +:sphinx_ok + + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\Consular.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\Consular.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdf" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdfja" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf-ja + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +if "%1" == "coverage" ( + %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage + if errorlevel 1 exit /b 1 + echo. + echo.Testing of coverage in the sources finished, look at the ^ +results in %BUILDDIR%/coverage/python.txt. + goto end +) + +if "%1" == "xml" ( + %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The XML files are in %BUILDDIR%/xml. + goto end +) + +if "%1" == "pseudoxml" ( + %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. + goto end +) + +:end diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..c3ab0d5 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,5 @@ +pytest +pytest-coverage +pytest-xdist +flake8 +sphinx diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..903109c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +Twisted +Klein +treq +click diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..050d01b --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[pytest] +addopts = --doctest-modules --verbose --ignore=ve/ consular diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..43a96fb --- /dev/null +++ b/setup.py @@ -0,0 +1,36 @@ +import os + +from setuptools import setup, find_packages + +here = os.path.abspath(os.path.dirname(__file__)) +with open(os.path.join(here, 'README.rst')) as f: + README = f.read() + +with open(os.path.join(here, 'requirements.txt')) as f: + requires = filter(None, f.readlines()) + +with open(os.path.join(here, 'VERSION')) as f: + version = f.read().strip() + +setup(name='consular', + version=version, + description='Consular', + long_description=README, + classifiers=[ + "Programming Language :: Python", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Internet :: WWW/HTTP :: WSGI :: Application", + ], + author='Simon de Haan', + author_email='simon@praekeltfoundation.org', + url='http://github.com/universalcore/consular', + license='BSD', + keywords='marathon,consul,mesos', + packages=find_packages(exclude=['docs']), + include_package_data=True, + zip_safe=False, + install_requires=requires, + tests_require=requires, + entry_points={ + 'console_scripts': ['consular = consular.cli:main'], + }) diff --git a/sideloader/deploy.sh b/sideloader/deploy.sh new file mode 100755 index 0000000..2fc4cfe --- /dev/null +++ b/sideloader/deploy.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +cp -a consular ./build/ diff --git a/sideloader/postinstall.sh b/sideloader/postinstall.sh new file mode 100755 index 0000000..759819d --- /dev/null +++ b/sideloader/postinstall.sh @@ -0,0 +1,8 @@ +pip="${VENV}/bin/pip" + +cd "${INSTALLDIR}/${REPO}/" + +$pip install -e "${INSTALLDIR}/${REPO}/" + +supervisorctl restart consular +supervisorctl restart consul-template diff --git a/templates/nginx.conf b/templates/nginx.conf new file mode 100644 index 0000000..6ffebab --- /dev/null +++ b/templates/nginx.conf @@ -0,0 +1,157 @@ +# NOTE: This is an nginx configuration automatically generated by +# consul-template, there's no point in making manual changes +# to this file. + + + +upstream gem-au-59 { + server qa-unicore-y:31244; + +} + +server { + listen 80; + server_name au.gem.qa-hub.unicore.io ; + + location / { + proxy_pass http://gem-au-59; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto http; + proxy_set_header Host $http_host; + client_max_body_size 10m; + } +} + + +upstream gem-cn-58 { + server qa-unicore-z:8012; + +} + +server { + listen 80; + server_name cn.gem.qa-hub.unicore.io ; + + location / { + proxy_pass http://gem-cn-58; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto http; + proxy_set_header Host $http_host; + client_max_body_size 10m; + } +} + + +upstream gem-de-55 { + server qa-unicore-z:8100; + +} + +server { + listen 80; + server_name de.gem.qa-hub.unicore.io ; + + location / { + proxy_pass http://gem-de-55; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto http; + proxy_set_header Host $http_host; + client_max_body_size 10m; + } +} + + +upstream gem-fr-56 { + server qa-unicore-y:8847; + +} + +server { + listen 80; + server_name fr.gem.qa-hub.unicore.io ; + + location / { + proxy_pass http://gem-fr-56; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto http; + proxy_set_header Host $http_host; + client_max_body_size 10m; + } +} + + +upstream gem-ru-60 { + server qa-unicore-z:31559; + +} + +server { + listen 80; + server_name ru.gem.qa-hub.unicore.io ; + + location / { + proxy_pass http://gem-ru-60; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto http; + proxy_set_header Host $http_host; + client_max_body_size 10m; + } +} + + +upstream gem-us-57 { + server qa-unicore-z:8151; + +} + +server { + listen 80; + server_name us.gem.qa-hub.unicore.io ; + + location / { + proxy_pass http://gem-us-57; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto http; + proxy_set_header Host $http_host; + client_max_body_size 10m; + } +} + + +upstream mama-hk-63 { + server qa-unicore-z:31924; + +} + +server { + listen 80; + server_name hk.mama.qa-hub.unicore.io ; + + location / { + proxy_pass http://mama-hk-63; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto http; + proxy_set_header Host $http_host; + client_max_body_size 10m; + } +} + + +upstream unfpa-zm-62 { + server qa-unicore-y:8616; + +} + +server { + listen 80; + server_name zm.unfpa.qa-hub.unicore.io ; + + location / { + proxy_pass http://unfpa-zm-62; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto http; + proxy_set_header Host $http_host; + client_max_body_size 10m; + } +} + diff --git a/templates/nginx.ctmpl b/templates/nginx.ctmpl new file mode 100644 index 0000000..88e3f67 --- /dev/null +++ b/templates/nginx.ctmpl @@ -0,0 +1,31 @@ +# NOTE: This is an nginx configuration automatically generated by +# consul-template, there's no point in making manual changes +# to this file. +# +# This expects a key to exist under "consular//domain" +# in Consul's KV store. This is used to map a domain to a service in +# Nginx. If the key does not exist the service isn't added to the +# list of services in the Nginx config. + +{{range services}}{{if key (print "consular/" .Name "/domain") }} + +upstream {{.Name}} { + {{range service .Name }}server {{.Address}}:{{.Port}}; + {{end}} +} + +server { + listen 80; + server_name {{key (print "consular/" .Name "/domain") | parseJSON }}; + + location / { + proxy_pass http://{{.Name}}; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto http; + proxy_set_header Host $http_host; + client_max_body_size 10m; + } +} +{{else}} +# Skipped service {{.Name}} as it does not have a KV domain entry. +{{end}}{{end}}