From 7a39af34ae8e575008c2e6086d7c3e173b753590 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Mon, 13 Jul 2015 07:39:28 +0200 Subject: [PATCH 01/21] reqs --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 082785c..b692945 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -Twisted +Klein +treq From 103bcf1af5dc59ca9330af1425e355e6d95dc6ff Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Mon, 13 Jul 2015 08:23:19 +0200 Subject: [PATCH 02/21] consular app setup + testing --- .gitignore | 1 + .travis.yml | 1 + consular/cli.py | 14 +++++++++++++ consular/main.py | 17 ++++++++++++++++ consular/tests/__init__.py | 0 consular/tests/test_main.py | 39 +++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 7 files changed, 73 insertions(+) create mode 100644 consular/cli.py create mode 100644 consular/main.py create mode 100644 consular/tests/__init__.py create mode 100644 consular/tests/test_main.py diff --git a/.gitignore b/.gitignore index fba6305..e500ab5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.egg-info *.pyc .coverage +_trial_temp/ diff --git a/.travis.yml b/.travis.yml index 3edbb5b..3959eb5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ language: python python: - "2.7" + - "pypy" install: - pip install --upgrade pip - pip install flake8 diff --git a/consular/cli.py b/consular/cli.py new file mode 100644 index 0000000..5e3bdfa --- /dev/null +++ b/consular/cli.py @@ -0,0 +1,14 @@ +import click + + +@click.command() +@click.option('--host', default='localhost') +@click.option('--port', default='7000', type=int) +@click.option('--consul', default='http://localhost:8500', + help='The Consul HTTP API') +@click.option('--marathon', default='http://localhost:8080', + help='The Marathon HTTP API') +def main(host, port, consul, marathon): + from consular.main import Consular + consular = Consular(consul, marathon) + consular.app.run(host, port) diff --git a/consular/main.py b/consular/main.py new file mode 100644 index 0000000..9d34964 --- /dev/null +++ b/consular/main.py @@ -0,0 +1,17 @@ +import json + +from klein import Klein + + +class Consular(object): + + app = Klein() + + def __init__(self, consul_endpoint, marathon_endpoint): + self.consul_endpoint = consul_endpoint + self.marathon_endpoint = marathon_endpoint + + @app.route('/') + def index(self, request): + request.setHeader('Content-Type', 'application/json') + return json.dumps([]) diff --git a/consular/tests/__init__.py b/consular/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py new file mode 100644 index 0000000..19fd44e --- /dev/null +++ b/consular/tests/test_main.py @@ -0,0 +1,39 @@ +from twisted.trial.unittest import TestCase +from twisted.web.server import Site +from twisted.internet import reactor +from twisted.internet.defer import inlineCallbacks +from twisted.web.client import HTTPConnectionPool + +from consular.main import Consular + +import treq + + +class ConsularTest(TestCase): + + def setUp(self): + self.consular = Consular( + 'http://localhost:8500', + 'http://localhost:8080', + ) + self.site = Site(self.consular.app.resource()) + self.listener = reactor.listenTCP(0, self.site, interface='localhost') + self.listener_port = self.listener.getHost().port + self.addCleanup(self.listener.loseConnection) + self.pool = HTTPConnectionPool(reactor, persistent=False) + self.addCleanup(self.pool.closeCachedConnections) + + def request(self, method, path): + return treq.request(method, 'http://localhost:%s%s' % ( + self.listener_port, + path + ), pool=self.pool) + + def tearDown(self): + pass + + @inlineCallbacks + def test_service(self): + response = yield self.request('GET', '/') + self.assertEqual(response.code, 200) + self.assertEqual((yield response.json()), []) diff --git a/requirements.txt b/requirements.txt index b692945..e4ffee2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ Klein treq +click From 1cd5898384dec64ef0327b622cecb076ca42df4c Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Mon, 13 Jul 2015 08:29:06 +0200 Subject: [PATCH 03/21] coveralls + comments --- .travis.yml | 1 + consular/tests/test_main.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 3959eb5..2c05241 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ python: - "2.7" - "pypy" install: + - pip install coveralls - pip install --upgrade pip - pip install flake8 - pip install -r requirements-dev.txt diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 19fd44e..7918dc4 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -16,10 +16,15 @@ def setUp(self): 'http://localhost:8500', 'http://localhost:8080', ) + + # spin up a site so we can test it, pretty sure Klein has better + # ways of doing this but they're not documented anywhere. self.site = Site(self.consular.app.resource()) self.listener = reactor.listenTCP(0, self.site, interface='localhost') self.listener_port = self.listener.getHost().port self.addCleanup(self.listener.loseConnection) + + # cleanup stuff for treq's global http request pool self.pool = HTTPConnectionPool(reactor, persistent=False) self.addCleanup(self.pool.closeCachedConnections) From 04b3acb11f47ec19b65d82502f3bddb3bdb85c72 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Mon, 13 Jul 2015 08:34:53 +0200 Subject: [PATCH 04/21] README badges --- README.rst | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/README.rst b/README.rst index 07e9edc..f1b0379 100644 --- a/README.rst +++ b/README.rst @@ -1,2 +1,46 @@ Consular ======== + +Receive events from Marathon_, update Consul_ with the relevant information +about services & tasks. + +.. image:: https://travis-ci.org/smn/consular.svg?branch=develop + :target: https://travis-ci.org/smn/consular + :alt: Continuous Integration + +.. image:: https://coveralls.io/repos/smn/consular/badge.png?branch=develop + :target: https://coveralls.io/r/smn/consular?branch=develop + :alt: Code Coverage + +.. image:: https://readthedocs.org/projects/consular/badge/?version=latest + :target: https://consular.readthedocs.org + :alt: Consular Documentation + +.. image:: https://pypip.in/version/consular/badge.svg + :target: https://pypi.python.org/pypi/consular + :alt: Pypi Package + +Usage +~~~~~ + +:: + + $ pip install consular + $ consular --help + + +Installing for local dev +~~~~~~~~~~~~~~~~~~~~~~~~ + +Make sure elasticsearch_ is running, then:: + + $ git clone https://github.com/smn/consular.git + $ cd consular + $ virtualenv ve + $ source ve/bin/activate + (ve)$ pip install -e . + (ve)$ pip install -r requirements-dev.txt + + +.. _Marathon: http://mesosphere.github.io/marathon/ +.. _Consul: http://consul.io/ From 0dc12163f5920f2cd8da46ab17ab9d63ce654996 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Mon, 13 Jul 2015 08:37:54 +0200 Subject: [PATCH 05/21] require sphinx --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 8591f05..c3ab0d5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,3 +2,4 @@ pytest pytest-coverage pytest-xdist flake8 +sphinx From 429b5ae2cc1e5175c74c751e1ec9df72463f269a Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Mon, 13 Jul 2015 08:40:34 +0200 Subject: [PATCH 06/21] docs templates --- .gitignore | 1 + docs/Makefile | 192 ++++++++++++++++++++++++++++++++ docs/conf.py | 292 +++++++++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 22 ++++ docs/make.bat | 263 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 770 insertions(+) create mode 100644 docs/Makefile create mode 100644 docs/conf.py create mode 100644 docs/index.rst create mode 100644 docs/make.bat diff --git a/.gitignore b/.gitignore index e500ab5..70345d0 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *.pyc .coverage _trial_temp/ +/docs/_build diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..14ee9bb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,192 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + +clean: + rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Consular.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Consular.qhc" + +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/Consular" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Consular" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..4b22d4f --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,292 @@ +# -*- coding: utf-8 -*- +# +# Consular documentation build configuration file, created by +# sphinx-quickstart on Mon Jul 13 08:39:08 2015. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os +import shlex + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.viewcode', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Consular' +copyright = u'2015, Simon de Haan' +author = u'Simon de Haan' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = open('../VERSION').read() +# The full version, including alpha/beta/rc tags. +release = version + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' +#html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +#html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +#html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'Consulardoc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', + +# Latex figure (float) alignment +#'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'Consular.tex', u'Consular Documentation', + u'Simon de Haan', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'consular', u'Consular Documentation', + [author], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'Consular', u'Consular Documentation', + author, 'Consular', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'https://docs.python.org/': None} diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..7df3b12 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,22 @@ +.. Consular documentation master file, created by + sphinx-quickstart on Mon Jul 13 08:39:08 2015. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to Consular's documentation! +==================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..7442659 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,263 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +set I18NSPHINXOPTS=%SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. xml to make Docutils-native XML files + echo. pseudoxml to make pseudoxml-XML files for display purposes + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + echo. coverage to run coverage check of the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + + +REM Check if sphinx-build is available and fallback to Python version if any +%SPHINXBUILD% 2> nul +if errorlevel 9009 goto sphinx_python +goto sphinx_ok + +:sphinx_python + +set SPHINXBUILD=python -m sphinx.__init__ +%SPHINXBUILD% 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +:sphinx_ok + + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\Consular.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\Consular.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdf" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdfja" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf-ja + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +if "%1" == "coverage" ( + %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage + if errorlevel 1 exit /b 1 + echo. + echo.Testing of coverage in the sources finished, look at the ^ +results in %BUILDDIR%/coverage/python.txt. + goto end +) + +if "%1" == "xml" ( + %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The XML files are in %BUILDDIR%/xml. + goto end +) + +if "%1" == "pseudoxml" ( + %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. + goto end +) + +:end From f1bbb49b64c09a4c7341fac0509756a8cd31e5a8 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Mon, 13 Jul 2015 08:50:57 +0200 Subject: [PATCH 07/21] document command line params --- consular/cli.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/consular/cli.py b/consular/cli.py index 5e3bdfa..064fd01 100644 --- a/consular/cli.py +++ b/consular/cli.py @@ -2,8 +2,10 @@ @click.command() -@click.option('--host', default='localhost') -@click.option('--port', default='7000', type=int) +@click.option('--host', default='localhost', + help='The host to bind to.') +@click.option('--port', default='7000', type=int, + help='The port to listen to.') @click.option('--consul', default='http://localhost:8500', help='The Consul HTTP API') @click.option('--marathon', default='http://localhost:8080', From c4bbc61e153be048d60e9c7fb256b0604168c596 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Mon, 13 Jul 2015 09:49:38 +0200 Subject: [PATCH 08/21] test for event updates --- consular/main.py | 44 +++++++++++++++++++++++++++++++++++ consular/tests/test_main.py | 46 ++++++++++++++++++++++++++++++++++--- 2 files changed, 87 insertions(+), 3 deletions(-) diff --git a/consular/main.py b/consular/main.py index 9d34964..9943982 100644 --- a/consular/main.py +++ b/consular/main.py @@ -1,5 +1,9 @@ import json +from twisted.internet import reactor +from twisted.web.client import HTTPConnectionPool + +import treq from klein import Klein @@ -10,8 +14,48 @@ class Consular(object): def __init__(self, consul_endpoint, marathon_endpoint): self.consul_endpoint = consul_endpoint self.marathon_endpoint = marathon_endpoint + self.pool = HTTPConnectionPool(reactor, persistent=False) + self.event_dispatch = { + 'status_update_event': self.handle_status_update_event, + } + self.unknown_event_handler = self.handle_unknown_event + + def consul_request(self, method, path, data): + return treq.request( + method, '%s%s' % (self.consul_endpoint, path), + headers={ + 'Content-Type': 'application/json', + }, + data=json.dumps(data), + pool=self.pool) @app.route('/') def index(self, request): request.setHeader('Content-Type', 'application/json') return json.dumps([]) + + @app.route('/events') + def events(self, request): + request.setHeader('Content-Type', 'application/json') + event = json.load(request.content) + handler = self.event_dispatch.get( + event.get('eventType'), self.handle_unknown_event) + return handler(request, event) + + def handle_status_update_event(self, request, event): + # NOTE: Marathon sends a list of ports, I don't know yet when & if + # there are multiple values in that list. + d = self.consul_request('PUT', '/v1/agent/service/register', { + "Name": event['appId'].rsplit('/', 1)[1], + "Address": event['host'], + "Port": event['ports'][0], + }) + d.addCallback(lambda _: json.dumps({'status': 'ok'})) + return d + + def handle_unknown_event(self, request, event): + request.setHeader('Content-Type', 'application/json') + request.setResponseCode(400) # bad request + return json.dumps({ + 'error': 'Event type %s not supported' % (event.get('eventType'),) + }) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 7918dc4..39a254c 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -1,7 +1,9 @@ +import json + from twisted.trial.unittest import TestCase from twisted.web.server import Site from twisted.internet import reactor -from twisted.internet.defer import inlineCallbacks +from twisted.internet.defer import inlineCallbacks, DeferredQueue, Deferred from twisted.web.client import HTTPConnectionPool from consular.main import Consular @@ -11,6 +13,8 @@ class ConsularTest(TestCase): + timeout = 5 + def setUp(self): self.consular = Consular( 'http://localhost:8500', @@ -28,11 +32,26 @@ def setUp(self): self.pool = HTTPConnectionPool(reactor, persistent=False) self.addCleanup(self.pool.closeCachedConnections) - def request(self, method, path): + # We use this to mock requests going to Consul + self.consul_requests = DeferredQueue() + + def mock_consul_request(method, path, data): + d = Deferred() + self.consul_requests.put({ + 'method': method, + 'path': path, + 'data': data, + 'deferred': d, + }) + return d + + self.patch(self.consular, 'consul_request', mock_consul_request) + + def request(self, method, path, data=None): return treq.request(method, 'http://localhost:%s%s' % ( self.listener_port, path - ), pool=self.pool) + ), data=(json.dumps(data) if data else None), pool=self.pool) def tearDown(self): pass @@ -42,3 +61,24 @@ def test_service(self): response = yield self.request('GET', '/') self.assertEqual(response.code, 200) self.assertEqual((yield response.json()), []) + + @inlineCallbacks + def test_status_update_event(self): + self.request('POST', '/events', { + "eventType": "status_update_event", + "timestamp": "2014-03-01T23:29:30.158Z", + "slaveId": "20140909-054127-177048842-5050-1494-0", + "taskId": "my-app_0-1396592784349", + "taskStatus": "TASK_RUNNING", + "appId": "/my-app", + "host": "slave-1234.acme.org", + "ports": [31372], + "version": "2014-04-04T06:26:23.051Z" + }) + request = yield self.consul_requests.get() + self.assertEqual(request['data'], { + 'Name': 'my-app', + 'Address': 'slave-1234.acme.org', + 'Port': 31372, + }) + request['deferred'].callback('ok') From 6c6b4c791bf61f9718f5f3520f2307cf9eff968d Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Mon, 13 Jul 2015 09:56:12 +0200 Subject: [PATCH 09/21] test for unsupported events --- consular/main.py | 2 +- consular/tests/test_main.py | 27 +++++++++++++++++++++++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/consular/main.py b/consular/main.py index 9943982..bc69546 100644 --- a/consular/main.py +++ b/consular/main.py @@ -57,5 +57,5 @@ def handle_unknown_event(self, request, event): request.setHeader('Content-Type', 'application/json') request.setResponseCode(400) # bad request return json.dumps({ - 'error': 'Event type %s not supported' % (event.get('eventType'),) + 'error': 'Event type %s not supported.' % (event.get('eventType'),) }) diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 39a254c..8f7eea3 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -48,10 +48,13 @@ def mock_consul_request(method, path, data): self.patch(self.consular, 'consul_request', mock_consul_request) def request(self, method, path, data=None): - return treq.request(method, 'http://localhost:%s%s' % ( - self.listener_port, - path - ), data=(json.dumps(data) if data else None), pool=self.pool) + return treq.request( + method, 'http://localhost:%s%s' % ( + self.listener_port, + path + ), + data=(json.dumps(data) if data is not None else None), + pool=self.pool) def tearDown(self): pass @@ -62,6 +65,22 @@ def test_service(self): self.assertEqual(response.code, 200) self.assertEqual((yield response.json()), []) + @inlineCallbacks + def test_handle_unknown_event(self): + response = yield self.request('POST', '/events', {'eventType': 'Foo'}) + data = yield response.json() + self.assertEqual(data, { + 'error': 'Event type Foo not supported.' + }) + + @inlineCallbacks + def test_handle_unspecified_event(self): + response = yield self.request('POST', '/events', {}) + data = yield response.json() + self.assertEqual(data, { + 'error': 'Event type None not supported.' + }) + @inlineCallbacks def test_status_update_event(self): self.request('POST', '/events', { From a05a8ad742fe5e1185a8a95588043ffb99d9d9ee Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Mon, 13 Jul 2015 09:58:23 +0200 Subject: [PATCH 10/21] cache pip --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 2c05241..d4f0692 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: python python: - "2.7" - "pypy" +cache: pip install: - pip install coveralls - pip install --upgrade pip From 09c3e0cab7e37571e025a94ba85ca10a7c191c0c Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Mon, 13 Jul 2015 10:13:26 +0200 Subject: [PATCH 11/21] update docs --- docs/index.rst | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 7df3b12..21b0057 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,14 +3,7 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to Consular's documentation! -==================================== - -Contents: - -.. toctree:: - :maxdepth: 2 - +.. include: ../README.rst Indices and tables @@ -19,4 +12,3 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` * :ref:`search` - From de41d2c1431a32d03f90c2e9d5d759d45c5fb2fb Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Mon, 13 Jul 2015 10:15:06 +0200 Subject: [PATCH 12/21] fix badge links & readme --- README.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index f1b0379..997efdf 100644 --- a/README.rst +++ b/README.rst @@ -4,12 +4,12 @@ Consular Receive events from Marathon_, update Consul_ with the relevant information about services & tasks. -.. image:: https://travis-ci.org/smn/consular.svg?branch=develop - :target: https://travis-ci.org/smn/consular +.. image:: https://travis-ci.org/universalcore/consular.svg?branch=develop + :target: https://travis-ci.org/universalcore/consular :alt: Continuous Integration -.. image:: https://coveralls.io/repos/smn/consular/badge.png?branch=develop - :target: https://coveralls.io/r/smn/consular?branch=develop +.. image:: https://coveralls.io/repos/universalcore/consular/badge.png?branch=develop + :target: https://coveralls.io/r/universalcore/consular?branch=develop :alt: Code Coverage .. image:: https://readthedocs.org/projects/consular/badge/?version=latest @@ -34,7 +34,7 @@ Installing for local dev Make sure elasticsearch_ is running, then:: - $ git clone https://github.com/smn/consular.git + $ git clone https://github.com/universalcore/consulari.git $ cd consular $ virtualenv ve $ source ve/bin/activate From 23a16bf65a70e25bd98cdadb6a55fd5cf498a830 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Mon, 13 Jul 2015 10:27:21 +0200 Subject: [PATCH 13/21] update docs --- README.rst | 17 ----------------- docs/index.rst | 52 ++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 42 insertions(+), 27 deletions(-) diff --git a/README.rst b/README.rst index 997efdf..ccc1116 100644 --- a/README.rst +++ b/README.rst @@ -27,20 +27,3 @@ Usage $ pip install consular $ consular --help - - -Installing for local dev -~~~~~~~~~~~~~~~~~~~~~~~~ - -Make sure elasticsearch_ is running, then:: - - $ git clone https://github.com/universalcore/consulari.git - $ cd consular - $ virtualenv ve - $ source ve/bin/activate - (ve)$ pip install -e . - (ve)$ pip install -r requirements-dev.txt - - -.. _Marathon: http://mesosphere.github.io/marathon/ -.. _Consul: http://consul.io/ diff --git a/docs/index.rst b/docs/index.rst index 21b0057..997efdf 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,14 +1,46 @@ -.. Consular documentation master file, created by - sphinx-quickstart on Mon Jul 13 08:39:08 2015. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +Consular +======== -.. include: ../README.rst +Receive events from Marathon_, update Consul_ with the relevant information +about services & tasks. +.. image:: https://travis-ci.org/universalcore/consular.svg?branch=develop + :target: https://travis-ci.org/universalcore/consular + :alt: Continuous Integration -Indices and tables -================== +.. image:: https://coveralls.io/repos/universalcore/consular/badge.png?branch=develop + :target: https://coveralls.io/r/universalcore/consular?branch=develop + :alt: Code Coverage -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` +.. image:: https://readthedocs.org/projects/consular/badge/?version=latest + :target: https://consular.readthedocs.org + :alt: Consular Documentation + +.. image:: https://pypip.in/version/consular/badge.svg + :target: https://pypi.python.org/pypi/consular + :alt: Pypi Package + +Usage +~~~~~ + +:: + + $ pip install consular + $ consular --help + + +Installing for local dev +~~~~~~~~~~~~~~~~~~~~~~~~ + +Make sure elasticsearch_ is running, then:: + + $ git clone https://github.com/universalcore/consulari.git + $ cd consular + $ virtualenv ve + $ source ve/bin/activate + (ve)$ pip install -e . + (ve)$ pip install -r requirements-dev.txt + + +.. _Marathon: http://mesosphere.github.io/marathon/ +.. _Consul: http://consul.io/ From 9d1f608a294e69b0fb0518f81c74dc2282bd1443 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Mon, 13 Jul 2015 10:30:10 +0200 Subject: [PATCH 14/21] fix badges --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 997efdf..d6c8809 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,7 +16,7 @@ about services & tasks. :target: https://consular.readthedocs.org :alt: Consular Documentation -.. image:: https://pypip.in/version/consular/badge.svg +.. image:: https://badge.fury.io/py/consular.svg :target: https://pypi.python.org/pypi/consular :alt: Pypi Package From 9155ab6ad9020dcc5f994eb5f044b0b4f0dcadf5 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Mon, 13 Jul 2015 10:31:17 +0200 Subject: [PATCH 15/21] more README updates --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index ccc1116..72e1529 100644 --- a/README.rst +++ b/README.rst @@ -16,7 +16,7 @@ about services & tasks. :target: https://consular.readthedocs.org :alt: Consular Documentation -.. image:: https://pypip.in/version/consular/badge.svg +.. image:: https://badge.fury.io/py/consular.svg :target: https://pypi.python.org/pypi/consular :alt: Pypi Package From a02c4a5cdb604571f2b48b377681e256140ed61d Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Mon, 13 Jul 2015 10:55:55 +0200 Subject: [PATCH 16/21] handle staging, running & killed stages --- consular/main.py | 33 +++++++++++++++++++++-- consular/tests/test_main.py | 54 ++++++++++++++++++++++++++++++++++--- 2 files changed, 81 insertions(+), 6 deletions(-) diff --git a/consular/main.py b/consular/main.py index bc69546..0385177 100644 --- a/consular/main.py +++ b/consular/main.py @@ -2,11 +2,16 @@ from twisted.internet import reactor from twisted.web.client import HTTPConnectionPool +from twisted.internet.defer import succeed import treq from klein import Klein +def get_appid(event): + return event['appId'].rsplit('/', 1)[1] + + class Consular(object): app = Klein() @@ -20,7 +25,7 @@ def __init__(self, consul_endpoint, marathon_endpoint): } self.unknown_event_handler = self.handle_unknown_event - def consul_request(self, method, path, data): + def consul_request(self, method, path, data=None): return treq.request( method, '%s%s' % (self.consul_endpoint, path), headers={ @@ -43,16 +48,40 @@ def events(self, request): return handler(request, event) def handle_status_update_event(self, request, event): + dispatch = { + 'TASK_STAGING': self.noop, + 'TASK_STARTING': self.noop, + 'TASK_RUNNING': self.update_task_running, + 'TASK_FINISHED': self.update_task_killed, + 'TASK_FAILED': self.update_task_killed, + 'TASK_KILLED': self.update_task_killed, + 'TASK_LOST': self.update_task_killed, + } + handler = dispatch.get(event['taskStatus']) + return handler(request, event) + + def noop(self, request, event): + return succeed(json.dumps({ + 'status': 'ok' + })) + + def update_task_running(self, request, event): # NOTE: Marathon sends a list of ports, I don't know yet when & if # there are multiple values in that list. d = self.consul_request('PUT', '/v1/agent/service/register', { - "Name": event['appId'].rsplit('/', 1)[1], + "Name": get_appid(event), "Address": event['host'], "Port": event['ports'][0], }) d.addCallback(lambda _: json.dumps({'status': 'ok'})) return d + def update_task_killed(self, request, event): + d = self.consul_request('PUT', '/v1/agent/service/deregister/%s' % ( + get_appid(event),)) + d.addCallback(lambda _: json.dumps({'status': 'ok'})) + return d + def handle_unknown_event(self, request, event): request.setHeader('Content-Type', 'application/json') request.setResponseCode(400) # bad request diff --git a/consular/tests/test_main.py b/consular/tests/test_main.py index 8f7eea3..7b26ae0 100644 --- a/consular/tests/test_main.py +++ b/consular/tests/test_main.py @@ -13,7 +13,7 @@ class ConsularTest(TestCase): - timeout = 5 + timeout = 1 def setUp(self): self.consular = Consular( @@ -35,7 +35,7 @@ def setUp(self): # We use this to mock requests going to Consul self.consul_requests = DeferredQueue() - def mock_consul_request(method, path, data): + def mock_consul_request(method, path, data=None): d = Deferred() self.consul_requests.put({ 'method': method, @@ -82,8 +82,25 @@ def test_handle_unspecified_event(self): }) @inlineCallbacks - def test_status_update_event(self): - self.request('POST', '/events', { + def test_TASK_STAGING(self): + response = yield self.request('POST', '/events', { + "eventType": "status_update_event", + "timestamp": "2014-03-01T23:29:30.158Z", + "slaveId": "20140909-054127-177048842-5050-1494-0", + "taskId": "my-app_0-1396592784349", + "taskStatus": "TASK_STAGING", + "appId": "/my-app", + "host": "slave-1234.acme.org", + "ports": [31372], + "version": "2014-04-04T06:26:23.051Z" + }) + self.assertEqual((yield response.json()), { + 'status': 'ok' + }) + + @inlineCallbacks + def test_TASK_RUNNING(self): + d = self.request('POST', '/events', { "eventType": "status_update_event", "timestamp": "2014-03-01T23:29:30.158Z", "slaveId": "20140909-054127-177048842-5050-1494-0", @@ -95,9 +112,38 @@ def test_status_update_event(self): "version": "2014-04-04T06:26:23.051Z" }) request = yield self.consul_requests.get() + self.assertEqual(request['method'], 'PUT') + self.assertEqual(request['path'], '/v1/agent/service/register') self.assertEqual(request['data'], { 'Name': 'my-app', 'Address': 'slave-1234.acme.org', 'Port': 31372, }) request['deferred'].callback('ok') + response = yield d + self.assertEqual((yield response.json()), { + 'status': 'ok' + }) + + @inlineCallbacks + def test_TASK_KILLED(self): + d = self.request('POST', '/events', { + "eventType": "status_update_event", + "timestamp": "2014-03-01T23:29:30.158Z", + "slaveId": "20140909-054127-177048842-5050-1494-0", + "taskId": "my-app_0-1396592784349", + "taskStatus": "TASK_KILLED", + "appId": "/my-app", + "host": "slave-1234.acme.org", + "ports": [31372], + "version": "2014-04-04T06:26:23.051Z" + }) + request = yield self.consul_requests.get() + self.assertEqual(request['method'], 'PUT') + self.assertEqual( + request['path'], '/v1/agent/service/deregister/my-app') + request['deferred'].callback('ok') + response = yield d + self.assertEqual((yield response.json()), { + 'status': 'ok' + }) From 5f3e8690a38cf5dfa399a612ac150f463512c888 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Mon, 13 Jul 2015 10:58:14 +0200 Subject: [PATCH 17/21] fix README --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index 72e1529..611c82f 100644 --- a/README.rst +++ b/README.rst @@ -27,3 +27,7 @@ Usage $ pip install consular $ consular --help + + +.. _Marathon: http://mesosphere.github.io/marathon/ +.. _Consul: http://consul.io/ From 398e8f280d4359d61843d2b2a82d0827bc6a028b Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Mon, 13 Jul 2015 11:23:44 +0200 Subject: [PATCH 18/21] update docs --- docs/index.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index d6c8809..c72deaf 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -32,9 +32,7 @@ Usage Installing for local dev ~~~~~~~~~~~~~~~~~~~~~~~~ -Make sure elasticsearch_ is running, then:: - - $ git clone https://github.com/universalcore/consulari.git + $ git clone https://github.com/universalcore/consular.git $ cd consular $ virtualenv ve $ source ve/bin/activate From f6284177644b80ec8f69567a0b4e9a44e616851a Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Mon, 13 Jul 2015 11:41:50 +0200 Subject: [PATCH 19/21] sideloader scripts --- .deploy.yaml | 9 +++++++++ sideloader/deploy.sh | 3 +++ sideloader/postinstall.sh | 5 +++++ 3 files changed, 17 insertions(+) create mode 100644 .deploy.yaml create mode 100755 sideloader/deploy.sh create mode 100755 sideloader/postinstall.sh diff --git a/.deploy.yaml b/.deploy.yaml new file mode 100644 index 0000000..3c2bcd8 --- /dev/null +++ b/.deploy.yaml @@ -0,0 +1,9 @@ +name: consular +user: ubuntu +buildscript: sideloader/deploy.sh +postinstall: sideloader/postinstall.sh +virtualenv_prefix: consular +pip: [] +dependencies: + - libssl-dev + - libffi-dev diff --git a/sideloader/deploy.sh b/sideloader/deploy.sh new file mode 100755 index 0000000..2fc4cfe --- /dev/null +++ b/sideloader/deploy.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +cp -a consular ./build/ diff --git a/sideloader/postinstall.sh b/sideloader/postinstall.sh new file mode 100755 index 0000000..b086812 --- /dev/null +++ b/sideloader/postinstall.sh @@ -0,0 +1,5 @@ +pip="${VENV}/bin/pip" + +cd "${INSTALLDIR}/${REPO}/" + +$pip install -e "${INSTALLDIR}/${REPO}/" From 29dcfe11a8f15bb1d387d656b0431acf3c539d11 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Mon, 13 Jul 2015 13:50:32 +0200 Subject: [PATCH 20/21] remove unnecessary mocking variable (thanks @rizziepit) --- consular/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/consular/main.py b/consular/main.py index 0385177..2bb5568 100644 --- a/consular/main.py +++ b/consular/main.py @@ -23,7 +23,6 @@ def __init__(self, consul_endpoint, marathon_endpoint): self.event_dispatch = { 'status_update_event': self.handle_status_update_event, } - self.unknown_event_handler = self.handle_unknown_event def consul_request(self, method, path, data=None): return treq.request( From 3e75f0b2551af63e413fb6bab9be955aac886fcc Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Mon, 13 Jul 2015 13:56:11 +0200 Subject: [PATCH 21/21] version bump --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 81340c7..6e8bf73 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.4 +0.1.0