Merge branch 'release/0.17.0'

CHANGELOG

@@ -2,6 +2,15 @@
 ChangeLog
 *********
 
+0.17.0 (2016-10-11)
+===================
+- Feature: WaterButler accepts configuration from the environment, overriding any file-based
+configuration.  This helps MFR integrate nicer in a docker-compose environment. (thanks, @icereval!)
+- Fix: Fix pdf presentation mode on Safari. (thanks, @darioncassel!)
+- Fix: Fix aiohttp crashing on gzipped HEAD requests.
+- Fix: Fix incorrect WB API usage metrics.
+- Code: Bump raven dependency to 5.27.0.
+
 0.16.0 (2016-09-13)
 ===================
 - Feature: MFR now does .sav conversion via pspp-convert instead of rpy2.

dev-requirements.txt

@@ -2,13 +2,11 @@
 
 -e git+https://github.com/centerforopenscience/aiohttpretty.git@0.0.2#egg=aiohttpretty
 colorlog==2.5.0
-flake8==2.3.0
+flake8==3.0.4
 ipdb
 mccabe
-pep8
+pydevd==0.0.6
 pyflakes
 pytest==2.8.2
 pytest-cov==2.2.0
 pyzmq==14.4.1
-
-

mfr/__init__.py

@@ -1,2 +1,2 @@
-__version__ = '0.16.0'
+__version__ = '0.17.0'
 __import__('pkg_resources').declare_namespace(__name__)

mfr/extensions/image/settings.py

@@ -1,9 +1,7 @@
-try:
-    from mfr import settings
-except ImportError:
-    settings = {}
+from mfr import settings
 
-config = settings.get('IMAGE_EXTENSION_CONFIG', {})
+
+config = settings.child('IMAGE_EXTENSION_CONFIG')
 
 EXPORT_TYPE = config.get('EXPORT_TYPE', 'jpeg')
 EXPORT_MAXIMUM_SIZE = config.get('EXPORT_MAXIMUM_SIZE', '1200x1200')

mfr/extensions/pdb/settings.py

@@ -1,10 +1,6 @@
-try:
-    from mfr import settings
-except ImportError:
-    settings = {}
-
-config = settings.get('PDB_EXTENSION_CONFIG', {})
+from mfr import settings
 
+config = settings.child('PDB_EXTENSION_CONFIG')
 
 OPTIONS = config.get('OPTIONS', {
     'width': 'auto',

mfr/extensions/pdf/static/web/viewer.js

@@ -1969,8 +1969,8 @@ var PresentationMode = {
       this.container.requestFullscreen();
     } else if (this.container.mozRequestFullScreen) {
       this.container.mozRequestFullScreen();
-    } else if (this.container.webkitRequestFullScreen) {
-      this.container.webkitRequestFullScreen(Element.ALLOW_KEYBOARD_INPUT);
+    } else if (this.container.webkitRequestFullscreen) {
+      this.container.webkitRequestFullscreen(Element.ALLOW_KEYBOARD_INPUT);
     } else if (this.container.msRequestFullscreen) {
       this.container.msRequestFullscreen();
     } else {

mfr/extensions/tabular/settings.py

@@ -1,17 +1,12 @@
+from mfr import settings
 from mfr.extensions.tabular import libs
 
 
-try:
-    from mfr import settings
-except ImportError:
-    settings = {}
+config = settings.child('TABULAR_EXTENSION_CONFIG')
 
-config = settings.get('TABULAR_EXTENSION_CONFIG', {})
-
-
-MAX_SIZE = config.get('MAX_SIZE', 10000)
-TABLE_WIDTH = config.get('TABLE_WIDTH', 700)
-TABLE_HEIGHT = config.get('TABLE_HEIGHT', 600)
+MAX_SIZE = int(config.get('MAX_SIZE', 10000))
+TABLE_WIDTH = int(config.get('TABLE_WIDTH', 700))
+TABLE_HEIGHT = int(config.get('TABLE_HEIGHT', 600))
 
 LIBS = config.get('LIBS', {
     '.csv': [libs.csv_stdlib],

mfr/extensions/unoconv/settings.py

@@ -1,11 +1,9 @@
 import os
 
-try:
-    from mfr import settings
-except ImportError:
-    settings = {}
+from mfr import settings
 
-config = settings.get('UNOCONV_EXTENSION_CONFIG', {})
+
+config = settings.child('UNOCONV_EXTENSION_CONFIG')
 
 UNOCONV_BIN = config.get('UNOCONV_BIN', '/usr/bin/unoconv')
 

mfr/providers/http/settings.py

@@ -1,6 +1,4 @@
-try:
-    from mfr import settings
-except ImportError:
-    settings = {}
+from mfr import settings
 
-config = settings.get('HTTP_PROVIDER_CONFIG', {})
+
+config = settings.child('HTTP_PROVIDER_CONFIG')

mfr/providers/osf/provider.py

@@ -7,6 +7,7 @@
 
 import furl
 import aiohttp
+from aiohttp.errors import ContentEncodingError
 
 from waterbutler.core import streams
 
@@ -64,16 +65,19 @@ async def metadata(self):
             metadata = await metadata_request.json()
         else:
             # URL is for WaterButler v1 API
-            self.metrics.add('metadata.wb_api', 'v0')
+            self.metrics.add('metadata.wb_api', 'v1')
             metadata_request = await self._make_request('HEAD', download_url)
             # To make changes to current code as minimal as possible
             try:
                 metadata = {'data': json.loads(metadata_request.headers['x-waterbutler-metadata'])['attributes']}
+                await metadata_request.release()
             except KeyError:
                 raise exceptions.MetadataError(
                     'Failed to fetch metadata. Received response code {}'.format(str(metadata_request.status)),
                     code=400)
-        await metadata_request.release()
+            except ContentEncodingError:
+                pass  # hack: aiohttp tries to unzip empty body when Content-Encoding is set
+
         self.metrics.add('metadata.raw', metadata)
 
         # e.g.,

mfr/providers/osf/settings.py

@@ -1,11 +1,6 @@
-try:
-    from mfr import settings
-except ImportError:
-    settings = {}
+from mfr import settings
 
-config = settings.get('OSF_PROVIDER_CONFIG', {})
 
-
-# BASE_URL = config.get('BASE_URL', 'http://localhost:5001/')
+config = settings.child('OSF_PROVIDER_CONFIG')
 
 MFR_IDENTIFYING_HEADER = config.get('MFR_IDENTIFYING_HEADER', 'X-Cos-Mfr-Render-Request')

mfr/server/settings.py

@@ -2,56 +2,52 @@
 
 import furl
 
+from mfr import settings
 
-try:
-    from mfr import settings
-except ImportError:
-    settings = {}
-
-config = settings.get('SERVER_CONFIG', {})
 
+config = settings.child('SERVER_CONFIG')
 
 STATIC_PATH = config.get('STATIC_PATH', os.path.join(os.path.dirname(__file__), 'static'))
 
 ADDRESS = config.get('ADDRESS', 'localhost')
 PORT = config.get('PORT', 7778)
 
-DEBUG = config.get('DEBUG', False)
+DEBUG = config.get_bool('DEBUG', False)
 
-SSL_CERT_FILE = config.get('SSL_CERT_FILE', None)
-SSL_KEY_FILE = config.get('SSL_KEY_FILE', None)
+SSL_CERT_FILE = config.get_nullable('SSL_CERT_FILE', None)
+SSL_KEY_FILE = config.get_nullable('SSL_KEY_FILE', None)
 
-XHEADERS = config.get('XHEADERS', False)
+XHEADERS = config.get_bool('XHEADERS', False)
 CORS_ALLOW_ORIGIN = config.get('CORS_ALLOW_ORIGIN', '*')
 
-CHUNK_SIZE = config.get('CHUNK_SIZE', 65536)  # 64KB
-MAX_BUFFER_SIZE = config.get('MAX_BUFFER_SIZE', 1024 * 1024 * 100)  # 100MB
+CHUNK_SIZE = int(config.get('CHUNK_SIZE', 65536))  # 64KB
+MAX_BUFFER_SIZE = int(config.get('MAX_BUFFER_SIZE', 1024 * 1024 * 100))  # 100MB
 
 PROVIDER_NAME = config.get('PROVIDER_NAME', 'osf')
 
-CACHE_ENABLED = config.get('CACHE_ENABLED', False)
+CACHE_ENABLED = config.get_bool('CACHE_ENABLED', False)
 CACHE_PROVIDER_NAME = config.get('CACHE_PROVIDER_NAME', 'filesystem')
 CACHE_PROVIDER_SETTINGS = config.get('CACHE_PROVIDER_SETTINGS', {'folder': '/tmp/mfr/'})
 CACHE_PROVIDER_CREDENTIALS = config.get('CACHE_PROVIDER_CREDENTIALS', {})
 
 LOCAL_CACHE_PROVIDER_SETTINGS = config.get('LOCAL_CACHE_PROVIDER_SETTINGS', {'folder': '/tmp/mfrlocalcache/'})
 
-ALLOWED_PROVIDER_DOMAINS = config.get('ALLOWED_PROVIDER_DOMAINS', ['http://localhost:5000/', 'http://localhost:7777/'])
+ALLOWED_PROVIDER_DOMAINS = config.get('ALLOWED_PROVIDER_DOMAINS', 'http://localhost:5000/ http://localhost:7777/').split(' ')
 ALLOWED_PROVIDER_NETLOCS = []
 for domain in ALLOWED_PROVIDER_DOMAINS:
     ALLOWED_PROVIDER_NETLOCS.append(furl.furl(domain).netloc)
 
 
-analytics_config = config.get('ANALYTICS', {})
+analytics_config = config.child('ANALYTICS')
 
-keen_config = analytics_config.get('KEEN', {})
+keen_config = analytics_config.child('KEEN')
 KEEN_API_BASE_URL = keen_config.get('API_BASE_URL', 'https://api.keen.io')
 KEEN_API_VERSION = keen_config.get('API_VERSION', '3.0')
 
-keen_private_config = keen_config.get('PRIVATE', {})
-KEEN_PRIVATE_PROJECT_ID = keen_private_config.get('PROJECT_ID', None)
-KEEN_PRIVATE_WRITE_KEY = keen_private_config.get('WRITE_KEY', None)
+keen_private_config = keen_config.child('PRIVATE')
+KEEN_PRIVATE_PROJECT_ID = keen_private_config.get_nullable('PROJECT_ID', None)
+KEEN_PRIVATE_WRITE_KEY = keen_private_config.get_nullable('WRITE_KEY', None)
 
-keen_public_config = keen_config.get('PUBLIC', {})
-KEEN_PUBLIC_PROJECT_ID = keen_public_config.get('PROJECT_ID', None)
-KEEN_PUBLIC_WRITE_KEY = keen_public_config.get('WRITE_KEY', None)
+keen_public_config = keen_config.child('PUBLIC')
+KEEN_PUBLIC_PROJECT_ID = keen_public_config.get_nullable('PROJECT_ID', None)
+KEEN_PUBLIC_WRITE_KEY = keen_public_config.get_nullable('WRITE_KEY', None)

mfr/settings.py

@@ -4,6 +4,80 @@
 import logging.config
 
 
+class SettingsDict(dict):
+    """Allow overriding on-disk config via environment variables.  Normal config is done with a
+    hierarchical dict::
+
+        "SERVER_CONFIG": {
+          "HOST": "http://localhost:7777"
+        }
+
+    ``HOST`` can be retrieved in the python code with::
+
+        config = SettingsDict(json.load('local-config.json'))
+        server_cfg = config.child('SERVER_CONFIG')
+        host = server_cfg.get('HOST')
+
+    To override a value, join all of the parent keys and the child keys with an underscore::
+
+        $ SERVER_CONFIG_HOST='http://foo.bar.com' invoke server
+
+    Nested dicts can be handled with the ``.child()`` method.  Config keys will be all parent keys
+    joined by underscores::
+
+        "SERVER_CONFIG": {
+          "ANALYTICS": {
+            "PROJECT_ID": "foo"
+          }
+        }
+
+    The corresponding envvar for ``PROJECT_ID`` would be ``SERVER_CONFIG_ANALYTICS_PROJECT_ID``.
+    """
+
+    def __init__(self, *args, parent=None, **kwargs):
+        self.parent = parent
+        super().__init__(*args, **kwargs)
+
+    def get(self, key, default=None):
+        """Fetch a config value for ``key`` from the settings.  First checks the env, then the
+        on-disk config.  If neither exists, returns ``default``."""
+        env = self.full_key(key)
+        if env in os.environ:
+            return os.environ.get(env)
+        return super().get(key, default)
+
+    def get_bool(self, key, default=None):
+        """Fetch a config value and interpret as a bool. Since envvars are always strings,
+        interpret '0' and the empty string as False and '1' as True.  Anything else is probably
+        an acceident, so die screaming."""
+        value = self.get(key, default)
+        if value in [False, 0, '0', '']:
+            retval = False
+        elif value in [True, 1, '1']:
+            retval = True
+        else:
+            raise Exception(
+                '{} should be a truthy value, but instead we got {}'.format(
+                    self.full_key(key), value
+                )
+            )
+        return retval
+
+    def get_nullable(self, key, default=None):
+        """Fetch a config value and interpret the empty string as None. Useful for external code
+        that expects an explicit None."""
+        value = self.get(key, default)
+        return None if value == '' else value
+
+    def full_key(self, key):
+        """The name of the envvar which corresponds to this key."""
+        return '{}_{}'.format(self.parent, key) if self.parent else key
+
+    def child(self, key):
+        """Fetch a sub-dict of the current dict."""
+        return SettingsDict(self.get(key, {}), parent=self.full_key(key))
+
+
 PROJECT_NAME = 'mfr'
 PROJECT_CONFIG_PATH = '~/.cos'
 
@@ -61,21 +135,21 @@
     config_path = '{}/{}-{}.json'.format(PROJECT_CONFIG_PATH, PROJECT_NAME, env)
 
 
-config = {}
+config = SettingsDict()
 config_path = os.path.expanduser(config_path)
 if not os.path.exists(config_path):
     logging.warning('No \'{}\' configuration file found'.format(config_path))
 else:
     with open(os.path.expanduser(config_path)) as fp:
-        config = json.load(fp)
+        config = SettingsDict(json.load(fp))
 
 
-def get(key, default):
-    return config.get(key, default)
+def child(key):
+    return config.child(key)
 
 
-logging_config = get('LOGGING', DEFAULT_LOGGING_CONFIG)
+logging_config = config.get('LOGGING', DEFAULT_LOGGING_CONFIG)
 logging.config.dictConfig(logging_config)
 
 
-SENTRY_DSN = get('SENTRY_DSN', None)
+SENTRY_DSN = config.get_nullable('SENTRY_DSN', None)

requirements.txt

@@ -3,7 +3,7 @@ chardet==2.3.0
 furl==0.4.2
 invoke==0.11.1
 mako==1.0.1
-raven==5.10.2
+raven==5.27.0
 stevedore==1.2.0
 tornado==4.3
 

tasks.py

@@ -59,5 +59,12 @@ def test(verbose=False):
 @task
 def server():
     monkey_patch()
+
+    if os.environ.get('REMOTE_DEBUG', None):
+        import pydevd
+        # e.g. '127.0.0.1:5678'
+        remote_parts = os.environ.get('REMOTE_DEBUG').split(':')
+        pydevd.settrace(remote_parts[0], port=int(remote_parts[1]), suspend=False, stdoutToServer=True, stderrToServer=True)
+
     from mfr.server.app import serve
     serve()