Merge pull request #88 from xsnippet/envvar-conf

Use envvar based configuration

setup.py

@@ -47,6 +47,7 @@ def find_packages(namespace):
         'cerberus >= 0.9.2',
         'motor >= 1.1',
         'python-jose >= 1.3.2',
+        'python-decouple >= 3.1',
         'werkzeug >= 0.11.4',
         'picobox >= 2.0',
     ],

tests/conftest.py

@@ -3,7 +3,6 @@
 import re
 
 import picobox
-import pkg_resources
 import pytest
 
 from xsnippet.api.application import create_app
@@ -13,15 +12,14 @@
 
 @pytest.fixture(scope='function')
 def testconf():
-    path = pkg_resources.resource_filename('xsnippet.api', 'default.conf')
-    conf = get_conf(path)
-    conf['auth'] = {'secret': 'SWORDFISH'}
+    conf = get_conf()
+    conf['AUTH_SECRET'] = 'SWORDFISH'
 
     # This flag exist to workaround permissions (which we currently lack of)
     # and test PUT/PATCH requests to the snippet. Once permissions are
     # implemented and the hack is removed from @checkpermissions decorator
     # in resources/snippets.py - this silly flag must be thrown away.
-    conf['test'] = {'sudo': True}
+    conf['_SUDO'] = True
     return conf
 
 

tests/middlewares/test_auth.py

@@ -18,7 +18,7 @@
 @pytest.fixture(scope='function')
 async def testapp(aiohttp_client):
     app = web.Application(middlewares=[
-        middlewares.auth.auth({'secret': 'SWORDFISH'}),
+        middlewares.auth.auth({'AUTH_SECRET': 'SWORDFISH'}),
     ])
 
     async def handler(request):

tests/resources/test_snippets.py

@@ -255,7 +255,7 @@ async def test_get_snippets(testapp, snippets, params, rv, link):
     ),
 ])
 async def test_get_snippets_bad_request(testapp, testconf, snippets, params, rv):
-    testconf['snippet']['syntaxes'] = '\n'.join(['python', 'clojure'])
+    testconf['SNIPPET_SYNTAXES'] = ['python', 'clojure']
 
     resp = await testapp.get('/v1/snippets', params=params)
 
@@ -530,7 +530,7 @@ async def test_get_snippets_pagination_not_found(testapp):
       'updated_at': pytest.regex('\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}')}),
 ])
 async def test_post_snippet(testapp, testconf, snippet, rv):
-    testconf['snippet']['syntaxes'] = 'python\nclojure'
+    testconf['SNIPPET_SYNTAXES'] = ['python', 'clojure']
 
     resp = await testapp.post('/v1/snippets', data=json.dumps(snippet))
 
@@ -577,7 +577,7 @@ async def test_post_snippet(testapp, testconf, snippet, rv):
     ),
 ])
 async def test_post_snippet_bad_request(snippet, rv, testapp, testconf):
-    testconf['snippet']['syntaxes'] = 'python\nclojure'
+    testconf['SNIPPET_SYNTAXES'] = ['python', 'clojure']
 
     resp = await testapp.post('/v1/snippets', data=json.dumps(snippet))
 
@@ -729,7 +729,7 @@ async def test_put_snippet_not_found(testapp):
     ),
 ])
 async def test_put_snippet_bad_request(snippet, rv, testapp, testconf, snippets):
-    testconf['snippet']['syntaxes'] = 'python\nclojure'
+    testconf['SNIPPET_SYNTAXES'] = ['python', 'clojure']
 
     resp = await testapp.put('/v1/snippets/%d' % snippets[0]['id'], data=json.dumps(snippet))
 
@@ -803,7 +803,7 @@ async def test_patch_snippet_not_found(testapp):
     ),
 ])
 async def test_patch_snippet_bad_request(snippet, rv, testapp, testconf, snippets):
-    testconf['snippet']['syntaxes'] = 'python\nclojure'
+    testconf['SNIPPET_SYNTAXES'] = ['python', 'clojure']
 
     resp = await testapp.patch('/v1/snippets/%d' % snippets[0]['id'], data=json.dumps(snippet))
 
@@ -817,7 +817,7 @@ async def test_patch_snippet_bad_request(snippet, rv, testapp, testconf, snippet
     'delete',
 ])
 async def test_snippet_update_is_not_exposed(method, testapp, testconf, snippets):
-    testconf.remove_option('test', 'sudo')
+    testconf.pop('_SUDO', None)
     request = getattr(testapp, method)
 
     snippet = {'content': 'test'}

tests/resources/test_syntaxes.py

@@ -22,12 +22,12 @@ async def test_get_syntaxes_default_conf(testapp):
     assert await resp.json() == []
 
 
-@pytest.mark.parametrize('syntaxes,expected', [
-    ('', []),
-    ('clojure\npython', ['clojure', 'python'])
+@pytest.mark.parametrize('syntaxes', [
+    [],
+    ['clojure', 'python'],
 ])
-async def test_get_syntaxes_overriden_conf(testapp, testconf, syntaxes, expected):
-    testconf['snippet']['syntaxes'] = syntaxes
+async def test_get_syntaxes_overriden_conf(testapp, testconf, syntaxes):
+    testconf['SNIPPET_SYNTAXES'] = syntaxes
 
     resp = await testapp.get(
         '/v1/syntaxes',
@@ -36,11 +36,11 @@ async def test_get_syntaxes_overriden_conf(testapp, testconf, syntaxes, expected
         }
     )
     assert resp.status == 200
-    assert await resp.json() == expected
+    assert await resp.json() == syntaxes
 
 
 async def test_get_syntaxes_overriden_conf_no_syntaxes(testapp, testconf):
-    testconf['snippet'].pop('syntaxes', None)
+    testconf['SNIPPET_SYNTAXES'] = []
 
     resp = await testapp.get(
         '/v1/syntaxes',

tests/test_conf.py

@@ -0,0 +1,65 @@
+"""Test xsnippet.api.conf module."""
+
+import textwrap
+
+from xsnippet.api.conf import get_conf
+
+
+def test_get_conf(monkeypatch):
+    monkeypatch.setenv('XSNIPPET_SERVER_HOST', '1.2.3.4')
+    monkeypatch.setenv('XSNIPPET_SERVER_PORT', 1234)
+    monkeypatch.setenv('XSNIPPET_SERVER_ACCESS_LOG_FORMAT', '%x %y')
+    monkeypatch.setenv('XSNIPPET_DATABASE_CONNECTION_URI', 'mongodb://42.42.42.42/test')
+    monkeypatch.setenv('XSNIPPET_SNIPPET_SYNTAXES', 'foo,bar')
+    monkeypatch.setenv('XSNIPPET_AUTH_SECRET', 'x$3cret')
+
+    assert get_conf() == {
+        'SERVER_HOST': '1.2.3.4',
+        'SERVER_PORT': 1234,
+        'SERVER_ACCESS_LOG_FORMAT': '%x %y',
+        'DATABASE_CONNECTION_URI': 'mongodb://42.42.42.42/test',
+        'SNIPPET_SYNTAXES': ['foo', 'bar'],
+        'AUTH_SECRET': 'x$3cret',
+    }
+
+
+def test_get_conf_envvar(tmpdir, monkeypatch):
+    tmpdir.join('test.conf').write_text(
+        textwrap.dedent('''
+            [server]
+            host = 1.2.3.4
+            port = 1234
+            access_log_format = %%x %%y
+
+            [database]
+            connection = mongodb://42.42.42.42/test
+
+            [snippet]
+            syntaxes = foo
+                       bar
+
+            [auth]
+            secret = x$3cret
+        '''),
+        encoding='utf-8')
+    monkeypatch.setenv('XSNIPPET_TEST_CONF', tmpdir.join('test.conf').strpath)
+
+    assert get_conf(envvar='XSNIPPET_TEST_CONF') == {
+        'SERVER_HOST': '1.2.3.4',
+        'SERVER_PORT': 1234,
+        'SERVER_ACCESS_LOG_FORMAT': '%x %y',
+        'DATABASE_CONNECTION_URI': 'mongodb://42.42.42.42/test',
+        'SNIPPET_SYNTAXES': ['foo', 'bar'],
+        'AUTH_SECRET': 'x$3cret',
+    }
+
+
+def test_get_conf_defaults():
+    assert get_conf() == {
+        'SERVER_HOST': '127.0.0.1',
+        'SERVER_PORT': 8000,
+        'SERVER_ACCESS_LOG_FORMAT': '%t %a "%r" %s %b %{User-Agent}i" %Tf',
+        'DATABASE_CONNECTION_URI': 'mongodb://localhost:27017/xsnippet',
+        'SNIPPET_SYNTAXES': [],
+        'AUTH_SECRET': '',
+    }

xsnippet/api/__main__.py

@@ -28,30 +28,28 @@ def main(args=sys.argv[1:]):
     logging.basicConfig()
     logging.getLogger('aiohttp').setLevel(logging.INFO)
 
-    conf = get_conf([
-        os.path.join(os.path.dirname(__file__), 'default.conf'),
-    ], envvar='XSNIPPET_API_SETTINGS')
+    conf = get_conf(envvar='XSNIPPET_API_SETTINGS')
     database = create_connection(conf)
 
     # The secret is required to sign issued JWT tokens, therefore it's crucial
     # to warn about importance of settings secret before going production. The
     # only reason why we don't enforce it's setting is because we want the app
     # to fly (at least for development purpose) using defaults.
-    if not conf['auth'].get('secret', ''):
+    if not conf.get('AUTH_SECRET', ''):
         warnings.warn(
             'Auth secret has not been provided. Please generate a long random '
             'secret before going to production.')
-        conf['auth']['secret'] = binascii.hexlify(os.urandom(32)).decode('ascii')
+        conf['AUTH_SECRET'] = binascii.hexlify(os.urandom(32)).decode('ascii')
 
     with picobox.push(picobox.Box()) as box:
         box.put('conf', conf)
         box.put('database', database)
 
         web.run_app(
             create_app(conf, database),
-            host=conf['server']['host'],
-            port=int(conf['server']['port']),
-            access_log_format=conf['server']['access_log_format']
+            host=conf['SERVER_HOST'],
+            port=conf['SERVER_PORT'],
+            access_log_format=conf['SERVER_ACCESS_LOG_FORMAT'],
         )
 
 

xsnippet/api/application.py

@@ -35,7 +35,7 @@ def create_app(conf, db):
 
     app = aiohttp.web.Application(
         middlewares=[
-            middlewares.auth.auth(conf['auth']),
+            middlewares.auth.auth(conf),
         ])
     app.router.add_routes(routes.v1)
     app.on_startup.append(functools.partial(database.setup, db=db))

xsnippet/api/conf.py

@@ -12,23 +12,24 @@
 import os
 import configparser
 
+import decouple
 
-def get_conf(paths, envvar=None):
+
+def get_conf(envvar=None):
     """Return one settings instance combines from different sources.
 
     The idea that lies behind that function is to gather config settings
     from different sources, including dynamic once pointed by passed
     environment variable (usually production overrides).
 
-    :param paths: a list of paths to configuration files
-    :type paths: [str]
-
     :param envvar: an environment variable that points to additional config
     :type envvar: str
 
-    :return: a configuration instance
-    :rtype: :class:`configparser.ConfigParser`
+    :return: a configuration dictionary
+    :rtype: :class:`dict`
     """
+    # TODO: Drop INI support and use configuration via environment vars only.
+
     # Due to limitations of INI standard, there's no way to have an option
     # which value is a list. Fortunately, since Python 3.5 we can provide
     # so called converters: an additional functions to convert desired
@@ -38,9 +39,74 @@ def get_conf(paths, envvar=None):
     }
 
     conf = configparser.ConfigParser(converters=converters)
-    conf.read(paths)
+    conf.read_dict({
+        'server': {
+            # IP ADDRESS TO LISTEN ON
+            #
+            # By default, only localhost interface is used for listening.
+            # That's why no requests from outer world will be handled. If you
+            # want to accept any incoming request on any interface, please
+            # change that value to '0.0.0.0'.
+            'host': decouple.config('XSNIPPET_SERVER_HOST', default='127.0.0.1'),
+
+            # PORT TO LISTEN ON
+            #
+            # In production you probably will choose a default HTTP port -
+            # '80'.  If you want to pick any random free port, just pass '0'.
+            'port': decouple.config('XSNIPPET_SERVER_PORT', default=8000, cast=int),
+
+            # ACCESS LOG FORMAT
+            #
+            # Note, that you have to use double % signs for escaping to work.
+            #
+            # %t  - datetime of the request
+            # %a  - remote address
+            # %r  - request status line
+            # %s  - response status code
+            # %b  - response size (bytes)
+            # %Tf - request time (seconds)
+            #
+            # When deployed behind a reverse proxy, consider using the value of
+            # headers like X-Real-IP or X-Forwarded-For instead of %a
+            'access_log_format': decouple.config(
+                'XSNIPPET_SERVER_ACCESS_LOG_FORMAT',
+                default='%t %a "%r" %s %b %{User-Agent}i" %Tf',
+                # ConfigParser supports string interpolation so we need to
+                # escape '%' before processing.
+                cast=lambda value: value.replace('%', '%%')),
+        },
+
+        'database': {
+            # DATABASE CONNECTION URI
+            #
+            # Supported backends: MongoDB only
+            'connection': decouple.config(
+                'XSNIPPET_DATABASE_CONNECTION_URI', default='mongodb://localhost:27017/xsnippet')
+        },
+
+        'snippet': {
+            'syntaxes': decouple.config(
+                'XSNIPPET_SNIPPET_SYNTAXES',
+                default='',
+                # Convert comma separated list retrieved from env variable
+                # to newline separated list ready to be processed by
+                # configparser.
+                cast=lambda value: value.replace(',', '\n')),
+        },
+
+        'auth': {
+            'secret': decouple.config('XSNIPPET_AUTH_SECRET', default=''),
+        },
+    })
 
     if envvar is not None and envvar in os.environ:
         conf.read(os.environ[envvar])
 
-    return conf
+    return {
+        'SERVER_HOST': conf.get('server', 'host'),
+        'SERVER_PORT': conf.getint('server', 'port'),
+        'SERVER_ACCESS_LOG_FORMAT': conf.get('server', 'access_log_format'),
+        'DATABASE_CONNECTION_URI': conf.get('database', 'connection'),
+        'SNIPPET_SYNTAXES': conf.getlist('snippet', 'syntaxes'),
+        'AUTH_SECRET': conf.get('auth', 'secret'),
+    }

xsnippet/api/database.py

@@ -25,7 +25,7 @@ def create_connection(conf):
     :return: a database connection
     :rtype: :class:`motor.motor_asyncio.AsyncIOMotorDatabase`
     """
-    mongo = AsyncIOMotorClient(conf['database']['connection'])
+    mongo = AsyncIOMotorClient(conf['DATABASE_CONNECTION_URI'])
 
     # get_default_database returns a database from the connection string
     db = mongo.get_default_database()