Merge branch 'feature/oathpit-python36' into feature/oathpit

 [ENG-481]
 Closes: #387

.gitignore

@@ -19,3 +19,4 @@ dist/
 build/
 .cache
 .mypy_cache
+newrelic.ini

.travis.yml

@@ -1,7 +1,6 @@
 language: python
 
 python:
-  - "3.5"
   - "3.6"
 
 sudo: false

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.5-slim-jessie
+FROM python:3.6-slim-buster
 
 RUN usermod -d /home www-data && chown www-data:www-data /home
 
@@ -16,34 +16,12 @@ RUN apt-get update \
         libffi-dev \
         python-dev \
         gnupg2 \
+        # grab gosu for easy step-down from root
+        gosu \
     && apt-get clean \
     && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/*
 
-# grab gosu for easy step-down from root
-ENV GOSU_VERSION 1.4
-RUN apt-get update \
-    && apt-get install -y \
-        curl \
-    && for key in \
-      # GOSU
-      B42F6819007F00F88E364FD4036A9C25BF357DD4 \
-    ; do \
-      gpg2 --no-tty --keyserver hkp://ipv4.pool.sks-keyservers.net:80 --recv-keys "$key" || \
-      gpg2 --no-tty --keyserver hkp://ha.pool.sks-keyservers.net:80 --recv-keys "$key" || \
-      gpg2 --no-tty --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" || \
-      gpg2 --no-tty --keyserver hkp://keyserver.pgp.com:80 --recv-keys "$key" \
-    ; done \
-    && curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \
-  	&& curl -o /usr/local/bin/gosu.asc -SL "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \
-  	&& gpg2 --verify /usr/local/bin/gosu.asc \
-  	&& rm /usr/local/bin/gosu.asc \
-  	&& chmod +x /usr/local/bin/gosu \
-    && apt-get clean \
-    && apt-get autoremove -y \
-        curl \
-    && rm -rf /var/lib/apt/lists/*
-
 RUN mkdir -p /code
 WORKDIR /code
 

README.md

@@ -12,15 +12,15 @@
 
 ### Compatibility
 
-WaterButler is compatible with Python 3.5 (tested up to 3.5.3) and 3.6.
+WaterButler is compatible with Python 3.6.
 
 ### Documentation
 
 Documentation available at https://waterbutler.readthedocs.io/en/latest/
 
 ### Setting up
 
-In order to run WaterButler, you must create a Python 3.5-based virtualenv for it.
+In order to run WaterButler, you must create a Python 3.6-based virtualenv for it.
 
 For MacOSX, you can install the latest version of Python3 using:
 
@@ -31,15 +31,15 @@ brew install python3
 For Ubuntu users:
 
 ```bash
-apt-get install python3.5
+apt-get install python3.6
 ```
 
-After completing the installation of Python 3.5, you must create a virtual environment. This can be done with the following commands:
+After completing the installation of Python 3.6, you must create a virtual environment. This can be done with the following commands:
 
 ```bash
 pip install virtualenv
 pip install virtualenvwrapper
-mkvirtualenv --python=python3.5 waterbutler
+mkvirtualenv --python=python3.6 waterbutler
 
 pip install setuptools=37.0.0
 pip install invoke==0.13.0
@@ -78,30 +78,13 @@ WaterButler configuration is done through a JSON file (`waterbutler-test.json`)
 mkdir ~/.cos
 ```
 
-A minimal config file would be:
-
-```json
-{
-  "SERVER_CONFIG": {
-    "DEBUG": false
-  }
-}
-```
-
-That flag is necessary because Python 3.5's asyncio [has a bug](https://bugs.python.org/issue25394) that is triggered by turning on debugging in Tornado. If you are upgrading from the 3.4-based WaterButler, change that setting in your `waterbutler-test.json`. If you do not, you will encounter this error:
-
-```
-TypeError: throw() takes 2 positional arguments but 4 were given
-```
-
 The data in `waterbutler-test.json` is used by the many Django-style `settings.py` files sprinkled about.  Most of these files define a top-level key that its specific configuration should be listed under.  For instance, if you wanted your local WaterButler server to listen on port 8989 instead of the default 7777, you would check the settings file for `waterbutler.server`.  That file looks for `HOST` and `DOMAIN` configuration keys under the `SERVER_CONFIG` top-level key.  Your configuration file would need to be updated to look like this:
 
 ```json
 {
   "SERVER_CONFIG": {
     "PORT": 8989,
-    "DOMAIN": "http://localhost:8989",
-    "DEBUG": false
+    "DOMAIN": "http://localhost:8989"
   }
 }
 ```
@@ -112,8 +95,7 @@ If you then wanted to update the GitHub commit message WaterButler submits when
 {
   "SERVER_CONFIG": {
     "PORT": 8989,
-    "DOMAIN": "http://localhost:8989",
-    "DEBUG": false
+    "DOMAIN": "http://localhost:8989"
   },
   "GITHUB_PROVIDER_CONFIG": {
     "DELETE_FILE_MESSAGE": "WaterButler deleted this. You're welcome."

dev-requirements.txt

@@ -1,15 +1,15 @@
 -r requirements.txt
 
-git+https://github.com/cslzchen/aiohttpretty.git@feature/aiohttp3
+git+https://github.com/CenterForOpenScience/aiohttpretty.git@develop
 colorlog==2.5.0
-flake8==3.0.4
-ipdb
-ipython
+flake8==3.5.0
+ipdb==0.12.2
+ipython==7.8.0
 mypy==0.580
 pydevd==0.0.6
-pyflakes
-pytest==2.8.2
-pytest-asyncio==0.3.0
-pytest-cov==2.2.0
-python-coveralls
-redis
+pyflakes==1.6.0
+pytest==5.2.1
+pytest-asyncio==0.10.0
+pytest-cov==2.8.1
+python-coveralls==2.9.3
+redis==3.3.8

docs/conf.py

@@ -53,9 +53,9 @@
 extlinks = {}
 
 intersphinx_mapping = {
-    'python': ('https://docs.python.org/3.5', None),
+    'python': ('https://docs.python.org/3.6', None),
     'tornado': ('http://www.tornadoweb.org/en/stable/', None),
-    'aiohttp': ('https://aiohttp.readthedocs.org/en/v0.18.2/', None),
+    'aiohttp': ('https://aiohttp.readthedocs.org/en/stable/', None),
 }
 
 on_rtd = os.environ.get('READTHEDOCS', None) == 'True'

requirements.txt

@@ -1,22 +1,21 @@
-agent==0.1.2
-aiohttp==3.5.4
+aiocontextvars==0.2.2  # recommended for sentry-sdk
+aiohttp==3.6.2
 git+https://github.com/felliott/boto.git@feature/gen-url-query-params-6#egg=boto
 celery==3.1.17
+certifi==2019.09.11
 furl==0.4.2
 google-auth==1.4.1
 humanfriendly==1.31
 invoke==0.13.0
 multidict==4.5.2
+newrelic==5.2.0.127
 oauthlib==0.7.2
 pyjwe==1.0.0
 pyjwt==1.4.0
 python-dateutil==2.5.3
 pytz==2017.2
-raven==5.27.0
+sentry-sdk==0.13.2
 setuptools==37.0.0
 stevedore==1.2.0
-tornado==5.1.1
+tornado==6.0.3
 xmltodict==0.9.0
-
-# Issue: certifi-2015.9.6.1 and 2015.9.6.2 fail verification (https://github.com/certifi/python-certifi/issues/26)
-certifi==2015.4.28

setup.cfg

@@ -10,7 +10,7 @@ exclude = .ropeproject,tests/*,src/*,env,venv
 [mypy]
 # These default options are aimed at a newly converted codebase. See:
 #   http://mypy.readthedocs.io/en/latest/command_line.html#ignore-missing-imports
-python_version = 3.5
+python_version = 3.6
 follow_imports = silent
 ignore_missing_imports = true
 incremental = true

setup.py

@@ -30,7 +30,7 @@ def parse_requirements(requirements):
         'Natural Language :: English',
         'Intended Audience :: Developers',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
         'Development Status :: 5 - Production/Stable',
         'License :: OSI Approved :: Apache Software License',
     ],

tasks.py

@@ -1,33 +1,16 @@
 import os
+import sys
 
 from invoke import task
 
 WHEELHOUSE_PATH = os.environ.get('WHEELHOUSE')
 
 
-def monkey_patch(ctx):
-    # Force an older cacert.pem from certifi v2015.4.28, prevents an ssl failure w/ identity.api.rackspacecloud.com.
-    #
-    # SubjectAltNameWarning: Certificate for identity.api.rackspacecloud.com has no `subjectAltName`, falling
-    # back to check for a `commonName` for now. This feature is being removed by major browsers and deprecated by
-    # RFC 2818. (See  https://github.com/shazow/urllib3/issues/497  for details.)
-    # SubjectAltNameWarning
-    import ssl
-    import certifi
-
-    _create_default_context = ssl.create_default_context
-
-    def create_default_context(purpose=ssl.Purpose.SERVER_AUTH, *, cafile=None, capath=None, cadata=None):
-        if cafile is None:
-            cafile = certifi.where()
-        return _create_default_context(purpose=purpose, cafile=cafile, capath=capath, cadata=cadata)
-    ssl.create_default_context = create_default_context
-
-
 @task
 def wheelhouse(ctx, develop=False, pty=True):
     req_file = 'dev-requirements.txt' if develop else 'requirements.txt'
-    cmd = 'pip wheel --find-links={} -r {} --wheel-dir={}'.format(WHEELHOUSE_PATH, req_file, WHEELHOUSE_PATH)
+    cmd = 'pip wheel --find-links={} -r {} --wheel-dir={}'.format(WHEELHOUSE_PATH, req_file,
+                                                                  WHEELHOUSE_PATH)
     ctx.run(cmd, pty=pty)
 
 
@@ -53,7 +36,8 @@ def flake(ctx):
 @task
 def mypy(ctx):
     """
-    Check python types using mypy (additional level of linting). Follows options defined in setup.cfg
+    Check python types using mypy (additional level of linting). Follows options defined in
+    setup.cfg
     """
     ctx.run('mypy waterbutler/', pty=True)
 
@@ -101,7 +85,6 @@ def test(ctx, verbose=False, types=False, nocov=False, provider=None, path=None)
 
 @task
 def celery(ctx, loglevel='INFO', hostname='%h'):
-    monkey_patch(ctx)
 
     from waterbutler.tasks.app import app
     command = ['worker']
@@ -119,13 +102,13 @@ def rabbitmq(ctx):
 
 @task
 def server(ctx):
-    monkey_patch(ctx)
 
     if os.environ.get('REMOTE_DEBUG', None):
         import pydevd
         # e.g. '127.0.0.1:5678'
         remote_parts = os.environ.get('REMOTE_DEBUG').split(':')
-        pydevd.settrace(remote_parts[0], port=int(remote_parts[1]), suspend=False, stdoutToServer=True, stderrToServer=True)
+        pydevd.settrace(remote_parts[0], port=int(remote_parts[1]), suspend=False,
+                        stdoutToServer=True, stderrToServer=True)
 
     from waterbutler.server.app import serve
     serve()
@@ -137,3 +120,28 @@ def clean(ctx, verbose=False):
     if verbose:
         print(cmd)
     ctx.run(cmd, pty=True)
+
+
+@task
+def newrelic_init(ctx, key=None, verbose=False):
+    if key is None:
+        sys.exit('No newrelic api key given.  Please generate one and rerun this command '
+                 'with `invoke newrelic_init --key=$key`')
+
+    cmd_tmpl = 'newrelic-admin generate-config {} newrelic.ini'
+    cmd = cmd_tmpl.format(key)
+    if verbose:
+        print(cmd_tmpl.format('<redacted>'))
+    ctx.run(cmd, pty=True)
+
+
+@task
+def newrelic_server(ctx, config='newrelic.ini', verbose=False):
+    if not os.path.exists(config):
+        sys.exit("Couldn't find config file '{}'.  Check path or run `invoke newrelic_init` "
+                 "to generate it.".format(config))
+
+    cmd = 'NEW_RELIC_CONFIG_FILE={} newrelic-admin run-program invoke server'.format(config)
+    if verbose:
+        print(cmd)
+    ctx.run(cmd, pty=True)

tests/conftest.py

@@ -17,13 +17,11 @@ def pytest_configure(config):
 
 
 def pytest_runtest_setup(item):
-    marker = item.get_marker('aiohttpretty')
-    if marker is not None:
+    if 'aiohttpretty' in item.keywords:
         aiohttpretty.clear()
         aiohttpretty.activate()
 
 
 def pytest_runtest_teardown(item, nextitem):
-    marker = item.get_marker('aiohttpretty')
-    if marker is not None:
+    if 'aiohttpretty' in item.keywords:
         aiohttpretty.deactivate()

tests/core/streams/fixtures.py

@@ -6,12 +6,10 @@
 from waterbutler.core.streams.http import ResponseStreamReader
 
 
-@pytest.fixture
 def mock_content():
     return type('mock_content', (object,), {'read': MockCoroutine(return_value=b'data')})
 
 
-@pytest.fixture
 def mock_content_eof():
     return type('mock_content_eof', (object,), {'read': MockCoroutine(return_value=None)})
 

tests/core/streams/test_base64encodestream.py

@@ -43,7 +43,7 @@ async def test_chunking(self):
 
             assert left_overs == b''
 
-    async def test_size(self):
+    def test_size(self):
         data = b'the ode to carp'
         expected = base64.b64encode(data)
         stream = streams.Base64EncodeStream(streams.StringStream(data))