From e08ffc354c2a968ebb2b3615a94aadca6f85daa1 Mon Sep 17 00:00:00 2001 From: Kofi Date: Tue, 8 Mar 2016 18:29:20 -0800 Subject: [PATCH 1/3] Add a discovery implementation in Python. --- .drone.yml | 7 ++++ Makefile | 16 +++++++++ discovery/__init__.py | 3 ++ discovery/discovery.py | 70 ++++++++++++++++++++++++++++++++++++++ pep8 | 4 +++ requirements-dev.txt | 2 ++ requirements.txt | 1 + setup.py | 35 +++++++++++++++++++ test/test_discovery.py | 77 ++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 215 insertions(+) create mode 100644 .drone.yml create mode 100644 Makefile create mode 100644 discovery/__init__.py create mode 100644 discovery/discovery.py create mode 100644 pep8 create mode 100644 requirements-dev.txt create mode 100644 requirements.txt create mode 100644 setup.py create mode 100644 test/test_discovery.py diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..59c40a6 --- /dev/null +++ b/.drone.yml @@ -0,0 +1,7 @@ +image: python2.7 +notify: + email: + recipients: + - drone@clever.com +script: + - make test diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4a1e3bc --- /dev/null +++ b/Makefile @@ -0,0 +1,16 @@ +.PHONY: all deps lint format test +SHELL := /bin/bash + +all: format lint test + +deps: + python setup.py develop + +format: + autopep8 -i -r -j0 -a --experimental --max-line-length 100 --indent-size 2 . + +lint: + pep8 --config ./pep8 . || true + +test: deps + nosetests ./test diff --git a/discovery/__init__.py b/discovery/__init__.py new file mode 100644 index 0000000..6c4415c --- /dev/null +++ b/discovery/__init__.py @@ -0,0 +1,3 @@ +from __future__ import absolute_import + +from .discovery import * diff --git a/discovery/discovery.py b/discovery/discovery.py new file mode 100644 index 0000000..5a98f3a --- /dev/null +++ b/discovery/discovery.py @@ -0,0 +1,70 @@ +import logger +import os + + +_ENV_VAR_TEMPLATE = 'SERVICE_{service_name}_{expose}_{component}' + + +def url(service_name, expose): + """ + Returns the URL for the given service and exposed interface, based on + environment variables of the form + `SERVICE_{SERVICE NAME}_{EXPOSE}_{PROTO|HOST|PORT}`. + """ + return '{}://{}'.format(proto(service_name, expose), host_port(service_name, expose)) + + +def host_port(service_name, expose): + """ + Returns a `{HOST}:{PORT}` string for the given service and exposed interface, + based on environment variables of the form + `SERVICE_{SERVICE NAME}_{EXPOSE}_{HOST|PORT}`. + """ + return '{}:{}'.format(host(service_name, expose), port(service_name, expose)) + + +def host(service_name, expose): + """ + Returns the host name for the given service and exposed interface, based on + environment variables of the form `SERVICE_{SERVICE NAME}_{EXPOSE}_{HOST}`. + """ + return _get_env_var(service_name, expose, 'HOST') + + +def port(service_name, expose): + """ + Returns the port for the given service and exposed interface, based on + environment variables of the form `SERVICE_{SERVICE NAME}_{EXPOSE}_{PROTO}`. + """ + return _get_env_var(service_name, expose, 'PORT') + + +def proto(service_name, expose): + """ + Returns the protocol for the given service and exposed interface, based on + environment variables of the form `SERVICE_{SERVICE NAME}_{EXPOSE}_{PROTO}`. + """ + return _get_env_var(service_name, expose, 'PROTO') + + +def _get_env_var(service_name, expose, component): + var_name = _ENV_VAR_TEMPLATE.format( + service_name=service_name, expose=expose, component=component) + var_name = var_name.upper().replace('-', '_') + + value = os.environ.get(var_name) + if (value is None): + raise MissingEnvironmentVariableError(var_name) + + return value + + +class MissingEnvironmentVariableError(Exception): + + """Raised when a required environment cannot be found.""" + + def __init__(self, var_name): + self.var_name = var_name + + def __str__(self): + return 'Missing environment variable `{}`.'.format(self.var_name) diff --git a/pep8 b/pep8 new file mode 100644 index 0000000..a885c8b --- /dev/null +++ b/pep8 @@ -0,0 +1,4 @@ +[pep8] +ignore = E111 +exclude = ./.eggs +max-line-length = 100 diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..0fb6450 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +-r requirements.txt +nose>=1.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2251373 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +kayvee==2.0.1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..a9894ef --- /dev/null +++ b/setup.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +import os +import sys +from setuptools import setup, find_packages +from pip.req import parse_requirements + +import pkg_resources + +here = os.path.abspath(os.path.dirname(__file__)) + +pr_kwargs = {} +if pkg_resources.get_distribution("pip").version >= '6.0': + pr_kwargs = {"session": False} + +install_reqs = parse_requirements( + os.path.join( + here, + './requirements.txt' if not sys.argv[1] in ['develop', 'test'] else './requirements-dev.txt' + ), **pr_kwargs) + +setup( + name='discovery', + version='0.1.0', + author='Clever (https://clever.com)', + author_email='tech-notify@clever.com', + url='https://github.com/Clever/discovery-python/', + packages=['discovery'], + install_requires=[str(ir.req) for ir in install_reqs], + setup_requires=['nose>=1.0'], + test_suite='nose.collector', + long_description="""\ + Programmatically find service endpoints. + """ +) diff --git a/test/test_discovery.py b/test/test_discovery.py new file mode 100644 index 0000000..8d28731 --- /dev/null +++ b/test/test_discovery.py @@ -0,0 +1,77 @@ +import discovery +import os +import unittest + +from discovery import MissingEnvironmentVariableError + + +class TestDiscovery(unittest.TestCase): + + def test_host_available(self): + host = 'host-available.ops.clever.com' + os.environ['SERVICE_HOST_AVAILABLE_TCP_HOST'] = host + + self.assertEqual(discovery.host('host-available', 'tcp'), host) + + def test_host_missing(self): + with self.assertRaisesRegexp(MissingEnvironmentVariableError, 'SERVICE_HOST_MISSING_HTTP_HOST'): + discovery.host('host-missing', 'http') + + def test_port_available(self): + port = '5000' + os.environ['SERVICE_PORT_AVAILABLE_TCP_PORT'] = port + + self.assertEqual(discovery.port('port-available', 'tcp'), port) + + def test_port_missing(self): + with self.assertRaisesRegexp(MissingEnvironmentVariableError, 'SERVICE_PORT_MISSING_HTTP_PORT'): + discovery.port('port-missing', 'http') + + def test_proto_available(self): + proto = 'http' + os.environ['SERVICE_PROTO_AVAILABLE_TCP_PROTO'] = proto + + self.assertEqual(discovery.proto('proto-available', 'tcp'), proto) + + def test_proto_missing(self): + with self.assertRaisesRegexp( + MissingEnvironmentVariableError, 'SERVICE_PROTO_MISSING_HTTP_PROTO'): + discovery.proto('proto-missing', 'http') + + def test_host_port_available(self): + host = 'host-port-available.ops.clever.com' + os.environ['SERVICE_HOST_PORT_AVAILABLE_HTTP_HOST'] = host + + port = '5000' + os.environ['SERVICE_HOST_PORT_AVAILABLE_HTTP_PORT'] = port + + self.assertEqual(discovery.host_port('host-port-available', 'http'), '{}:{}'.format(host, port)) + + def test_host_port_missing_port(self): + host = 'port-missing.ops.clever.com' + os.environ['SERVICE_PORT_MISSING_HTTP_HOST'] = host + + with self.assertRaisesRegexp(MissingEnvironmentVariableError, 'SERVICE_PORT_MISSING_HTTP_PORT'): + discovery.host_port('port-missing', 'http') + + def test_url_available(self): + proto = 'http' + os.environ['SERVICE_URL_AVAILABLE_BLAH_PROTO'] = proto + + host = 'user:pasws@url-available.ops.clever.com' + os.environ['SERVICE_URL_AVAILABLE_BLAH_HOST'] = host + + port = '9090' + os.environ['SERVICE_URL_AVAILABLE_BLAH_PORT'] = port + + self.assertEqual(discovery.url('url-available', 'blah'), '{}://{}:{}'.format(proto, host, port)) + + def test_url_missing_host(self): + proto = 'tcp' + os.environ['SERVICE_PROTO_MISSING_TCP_PROTO'] = proto + + port = '9090' + os.environ['SERVICE_PROTO_MISSING_TCP_PORT'] = port + + with self.assertRaisesRegexp(MissingEnvironmentVariableError, 'SERVICE_PROTO_MISSING_TCP_HOST'): + discovery.url('proto-missing', 'tcp') From a1e800552755514a30b0efb7b5d1f7fc6cdd41bb Mon Sep 17 00:00:00 2001 From: Kofi Date: Tue, 8 Mar 2016 18:46:39 -0800 Subject: [PATCH 2/3] Update README. --- README.md | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d8259da..ee24137 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,71 @@ # discovery-python -Programmatically find service endpoints (i.e. discovery-go for python) + +Programmatically find service endpoints. + +This library currently is just an abstraction around reading environment variables used for dependent services. + +## Installation + +### pip + +```sh +pip install git+https://@github.com/Clever/discovery-python.git@ +``` + +The Github token can be found in [dev-passwords](https://github.com/Clever/clever-ops/tree/master/credentials). + +### setup.py + +```python +from setuptools import setup + +# Assuming discovery v0.1.0 is being installed: +setup( + + # ... + + install_requires=['discovery==0.1.0'], + dependency_links=[ + 'https://@github.com/Clever/discovery-python/tarball/v0.1.0#egg=discovery-0.1.0' + ], + + # ... + +) +``` + +The Github token can be found in [dev-passwords](https://github.com/Clever/clever-ops/tree/master/credentials). + +## Usage + +```python +import discovery + +try: + redis_url = discovery.url('redis', 'tcp') + + redis_host_and_port = discovery.host_port('redis', 'tcp') + + redis_host = discovery.host('redis', 'tcp') + + redis_port = discovery.port('redis', 'tcp') + +except discovery.MissingEnvironmentVariableError as e: + print 'ERROR: Redis discovery failed: {}.'.format(e) + +``` + +## Environment Variables + +This library currently requires environment variables to be defined in the following format: + +``` +SERVICE_{SERVICE_NAME}_{EXPOSE}_{PROTO|HOST|PORT} +``` + +### Example: +```bash +SERVICE_REDIS_TCP_PROTO = "tcp" +SERVICE_REDIS_TCP_HOST = "localhost" +SERVICE_REDIS_TCP_PORT = "6379" +``` From 11b7c965fdd7b1b85350c213b7ce218cc3e400c3 Mon Sep 17 00:00:00 2001 From: Kofi Date: Fri, 11 Mar 2016 12:31:32 -0800 Subject: [PATCH 3/3] Add a publish script for version tagging. --- CHANGES.md | 2 ++ Makefile | 8 +++++++- README.md | 13 +++++++++++++ VERSION | 1 + publish.sh | 19 +++++++++++++++++++ setup.py | 5 ++++- 6 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 CHANGES.md create mode 100644 VERSION create mode 100755 publish.sh diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 0000000..b643dba --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,2 @@ +## 0.1.0 (2016-03-11) +- Initial version. diff --git a/Makefile b/Makefile index 4a1e3bc..6100a15 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,11 @@ -.PHONY: all deps lint format test +.PHONY: all clean deps format lint publish test SHELL := /bin/bash all: format lint test +clean: + find -type f -name '*.pyc' -delete + deps: python setup.py develop @@ -12,5 +15,8 @@ format: lint: pep8 --config ./pep8 . || true +publish: + ./publish.sh + test: deps nosetests ./test diff --git a/README.md b/README.md index ee24137..82e5796 100644 --- a/README.md +++ b/README.md @@ -69,3 +69,16 @@ SERVICE_REDIS_TCP_PROTO = "tcp" SERVICE_REDIS_TCP_HOST = "localhost" SERVICE_REDIS_TCP_PORT = "6379" ``` + +## Development + +### Publishing a new version + +1. Bump the version in the `VERSION` file and update the changelog in `CHANGES.md`. +2. Merge your changes into `master`. +3. Checkout `master` +4. Run the publish script: + + ```sh + ./publish.sh + ``` diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..6e8bf73 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.1.0 diff --git a/publish.sh b/publish.sh new file mode 100755 index 0000000..24ab2ef --- /dev/null +++ b/publish.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Creates git tags! +# Reads version from VERSION file. + +version=`cat VERSION` +changelog=CHANGES.md +grep $version $changelog >> /dev/null +if [[ $? -ne 0 ]]; then + echo "Couldn't find version $version in $changelog" + exit +fi + +read -p "Tag as v$version? [y/n] " -n 1 -r +if [[ $REPLY =~ ^[Yy]$ ]]; then + # create git tags + git tag -a v$version -m "version $version" + git push --tags +fi diff --git a/setup.py b/setup.py index a9894ef..70dfde6 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,9 @@ here = os.path.abspath(os.path.dirname(__file__)) +with open(os.path.join(here, 'VERSION')) as f: + VERSION = f.read().strip() + pr_kwargs = {} if pkg_resources.get_distribution("pip").version >= '6.0': pr_kwargs = {"session": False} @@ -21,7 +24,7 @@ setup( name='discovery', - version='0.1.0', + version=VERSION, author='Clever (https://clever.com)', author_email='tech-notify@clever.com', url='https://github.com/Clever/discovery-python/',