Merge pull request #802 from aaxelb/fix/upgrade-downgrade

[fix] upgrade/downgrade for compatibile dependencies

.github/workflows/run_tests.yml

@@ -13,8 +13,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.11']  # TODO: 3.12
-        postgres-version: ['10', '11', '12', '13', '14', '15']
+        python-version: ['3.10', '3.11']  # TODO: 3.12
+        postgres-version: ['10', '15']
     runs-on: ubuntu-latest
     services:
       postgres:

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11-slim as app
+FROM python:3.10-slim as app
 
 RUN apt-get update \
     && apt-get install -y \

api/middleware.py

@@ -1,55 +1,11 @@
-from io import StringIO
-import cProfile
-import pstats
-
 from django.conf import settings
 from django.http import HttpResponse
 
-from raven.contrib.django.raven_compat.models import client as sentry_client
+import sentry_sdk
 
 from api.deprecation import get_view_func_deprecation_level, DeprecationLevel
 
 
-# Adapted from http://www.djangosnippets.org/snippets/186/
-# Original author: udfalkso
-# Modified by: Shwagroo Team and Gun.io
-# Modified by: COS
-class ProfileMiddleware:
-    """
-    Displays hotshot profiling for any view.
-    http://yoursite.com/yourview/?prof
-    Add the "prof" key to query string by appending ?prof (or &prof=)
-    and you'll see the profiling results in your browser.
-    It's set up to only be available in django's debug mode, is available for superuser otherwise,
-    but you really shouldn't add this middleware to any production configuration.
-    """
-    def __init__(self, get_response):
-        self.get_response = get_response
-
-    def __call__(self, request):
-        if not ((settings.DEBUG or request.user.is_superuser) and 'prof' in request.GET):
-            return self.get_response(request)
-
-        request.GET._mutable = True
-        request.GET.pop('prof')
-        request.GET._mutable = False
-
-        prof = cProfile.Profile()
-        prof.enable()
-
-        response = self.get_response(request)
-
-        prof.disable()
-
-        s = StringIO()
-        ps = pstats.Stats(prof, stream=s).sort_stats('cumtime')
-        ps.print_stats()
-
-        response.content = s.getvalue()
-
-        return response
-
-
 class DeprecationMiddleware:
     def __init__(self, get_response):
         self.get_response = get_response
@@ -61,14 +17,13 @@ def process_view(self, request, view_func, view_args, view_kwargs):
         deprecation_level = get_view_func_deprecation_level(view_func)
 
         if deprecation_level in (DeprecationLevel.LOGGED, DeprecationLevel.HIDDEN):
-            sentry_client.captureMessage('Deprecated view usage', data={
-                'request': {
-                    'path': request.path,
-                    'method': request.method,
-                },
-                'deprecation_level': deprecation_level,
-                'HIDE_DEPRECATED_VIEWS': settings.HIDE_DEPRECATED_VIEWS,
-            })
+            sentry_sdk.capture_message('\n\t'.join((
+                'Deprecated view usage:',
+                f'request.path: {request.path}',
+                f'request.method: {request.method}',
+                f'deprecation_level: {deprecation_level}',
+                f'HIDE_DEPRECATED_VIEWS: {settings.HIDE_DEPRECATED_VIEWS}',
+            )))
 
         if settings.HIDE_DEPRECATED_VIEWS and deprecation_level == DeprecationLevel.HIDDEN:
             return HttpResponse(

api/normalizeddata/views.py

@@ -1,21 +1,19 @@
 import logging
+
 from django.db import transaction
 from django.urls import reverse
-
-from raven.contrib.django.raven_compat.models import client as sentry_client
-
 from rest_framework import status
 from rest_framework import generics
 from rest_framework.exceptions import ValidationError
 from rest_framework.response import Response
+import sentry_sdk
 
 from share import models
 from share.tasks import ingest
 from share.util import IDObfuscator
 from share.util.graph import MutableGraph
 from share.util.osf import guess_osf_guid
 from share.ingest.ingester import Ingester
-
 from api.base.views import ShareViewSet
 from api.normalizeddata.serializers import BasicNormalizedDataSerializer
 from api.normalizeddata.serializers import FullNormalizedDataSerializer
@@ -73,16 +71,8 @@ def get_queryset(self):
     def create(self, request, *args, **kwargs):
         try:
             return self._do_create(request, *args, **kwargs)
-        except Exception as e:
-            sentry_client.captureMessage('Bad normalizeddatum?', data={
-                'request': {
-                    'path': request.path,
-                    'method': request.method,
-                    'user': request.user,
-                },
-            }, extra={
-                'error': str(e),
-            })
+        except Exception:
+            sentry_sdk.capture_exception()  # get some insight into common validation errors
             raise
 
     def _do_create(self, request, *args, **kwargs):

api/views/feeds.py

@@ -1,14 +1,14 @@
 from xml.sax.saxutils import unescape
-from furl import furl
 import json
 import logging
-import pendulum
 
 from django.contrib.syndication.views import Feed
 from django.http import HttpResponseGone
 from django.utils.feedgenerator import Atom1Feed
 from django.conf import settings
-from raven.contrib.django.raven_compat.models import client as sentry_client
+from furl import furl
+import pendulum
+import sentry_sdk
 
 from share.search import IndexStrategy
 from share.search.exceptions import IndexStrategyError
@@ -69,7 +69,7 @@ def items(self, obj):
                 request_body=obj,
             )
         except IndexStrategyError:
-            sentry_client.captureException()
+            sentry_sdk.capture_exception()
             return
 
         def get_item(hit):

docker-compose.yml

@@ -147,15 +147,15 @@ services:
       - apt-get update &&
         apt-get install -y gcc &&
         pip install -r requirements.txt -r dev-requirements.txt &&
-        (python3 -m compileall /usr/local/lib/python3.11 || true) &&
-        rm -Rf /python3.11/* &&
+        (python3 -m compileall /usr/local/lib/python3.10 || true) &&
+        rm -Rf /python3.10/* &&
         apt-get remove -y gcc &&
-        cp -Rf -p /usr/local/lib/python3.11 / &&
+        cp -Rf -p /usr/local/lib/python3.10 / &&
         python3 setup.py develop
     restart: 'no'
     volumes:
       - ./:/code:cached
-      - share_requirements_vol:/python3.11
+      - share_requirements_vol:/python3.10
 
   frontend:
     image: quay.io/centerforopenscience/share-web:develop-local
@@ -177,7 +177,7 @@ services:
       - indexer
     volumes:
       - ./:/code:cached
-      - share_requirements_vol:/usr/local/lib/python3.11
+      - share_requirements_vol:/usr/local/lib/python3.10
       - elastic8_cert_vol:/elastic8_certs
     env_file:
       - .docker-compose.env
@@ -198,7 +198,7 @@ services:
       - frontend
     volumes:
       - ./:/code:cached
-      - share_requirements_vol:/usr/local/lib/python3.11
+      - share_requirements_vol:/usr/local/lib/python3.10
       - elastic8_cert_vol:/elastic8_certs
     env_file:
       - .docker-compose.env
@@ -216,7 +216,7 @@ services:
       - elastic8
     volumes:
       - ./:/code:cached
-      - share_requirements_vol:/usr/local/lib/python3.11
+      - share_requirements_vol:/usr/local/lib/python3.10
       - elastic8_cert_vol:/elastic8_certs
     env_file:
       - .docker-compose.env

project/celery.py

@@ -2,28 +2,13 @@
 
 import celery
 
-from raven.contrib.celery import register_signal, register_logger_signal
-
 # set the default Django settings module for the 'celery' program.
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'project.settings')
 
 from django.conf import settings  # noqa
 
 
-class Celery(celery.Celery):
-
-    def on_configure(self):
-        # Import has to be relative. "client" is not actually lazily initialized
-        from raven.contrib.django.raven_compat.models import client
-
-        if not client.is_enabled():
-            return
-
-        register_signal(client)
-        register_logger_signal(client)
-
-
-app = Celery('project')
+app = celery.Celery('project')
 
 app.config_from_object('django.conf:settings', namespace='CELERY')
 app.autodiscover_tasks([

project/logging_formatter.py

@@ -0,0 +1,10 @@
+import json
+import logging
+
+
+class JsonLogFormatter(logging.Formatter):
+    def format(self, record):
+        return json.dumps({
+            'severity': record.levelname,
+            'message': record.__dict__,
+        })

project/settings.py

@@ -57,6 +57,7 @@ def split(string, delim):
 DEBUG = bool(os.environ.get('DEBUG', True))
 
 VERSION = __version__
+GIT_COMMIT = os.environ.get('GIT_COMMIT', None)
 
 ALLOWED_HOSTS = [h for h in os.environ.get('ALLOWED_HOSTS', '').split(' ') if h]
 
@@ -236,15 +237,29 @@ def split(string, delim):
 
 if DEBUG:
     AUTH_PASSWORD_VALIDATORS = []
-# else:
+
 if os.environ.get('USE_SENTRY'):
-    INSTALLED_APPS += [
-        'raven.contrib.django.raven_compat',
-    ]
-    RAVEN_CONFIG = {
-        'dsn': os.environ.get('SENTRY_DSN', None),
-        'release': os.environ.get('GIT_COMMIT', None),
-    }
+    import sentry_sdk
+    from sentry_sdk.integrations.django import DjangoIntegration
+    sentry_sdk.init(
+        dsn=os.environ.get('SENTRY_DSN', None),
+        release=(
+            '-'.join((VERSION, GIT_COMMIT))
+            if GIT_COMMIT
+            else VERSION
+        ),
+        send_default_pii=False,
+        request_bodies=False,
+        debug=DEBUG,
+        integrations=[
+            DjangoIntegration(
+                transaction_style='endpoint',
+                middleware_spans=True,
+                signals_spans=True,
+                cache_spans=True,
+            ),
+        ],
+    )
 
 
 # TODO REMOVE BEFORE PRODUCTION
@@ -442,7 +457,7 @@ def route_urgent_task(name, args, kwargs, options, task=None, **kw):
 
 
 # Logging
-LOG_LEVEL = os.environ.get('LOG_LEVEL', 'WARNING').upper()
+LOG_LEVEL = os.environ.get('LOG_LEVEL', default=('INFO' if DEBUG else 'WARNING')).upper()
 
 LOGGING = {
     'version': 1,
@@ -451,61 +466,30 @@ def route_urgent_task(name, args, kwargs, options, task=None, **kw):
         'console': {
             '()': 'colorlog.ColoredFormatter',
             'format': '%(cyan)s[%(asctime)s]%(purple)s[%(threadName)s]%(log_color)s[%(levelname)s][%(name)s]: %(reset)s%(message)s'
-        }
+        },
+        'json': {
+            '()': 'project.logging_formatter.JsonLogFormatter',
+        },
     },
     'handlers': {
-        'sentry': {
-            'level': 'ERROR',  # To capture more than ERROR, change to WARNING, INFO, etc.
-            'class': 'raven.contrib.django.raven_compat.handlers.SentryHandler',
-            'tags': {'custom-tag': 'x'},
-        },
-        'console': {
+        'stream-to-stderr': {
             'class': 'logging.StreamHandler',
-            'level': 'DEBUG',
-            'formatter': 'console'
+            'level': LOG_LEVEL,
+            'formatter': ('console' if DEBUG else 'json'),
         },
     },
     'loggers': {
         '': {
-            'handlers': ['console'],
-            'level': 'INFO',
-            'propagate': False
-        },
-        'bots': {
-            'handlers': ['console'],
-            'level': LOG_LEVEL,
-            'propagate': False
-        },
-        'elasticsearch': {
-            'handlers': ['console'],
-            'level': LOG_LEVEL,
-            'propagate': False
-        },
-        'share': {
-            'handlers': ['console'],
             'level': LOG_LEVEL,
+            'handlers': ['stream-to-stderr'],
             'propagate': False
         },
         'django.db.backends': {
             'level': 'ERROR',
-            'handlers': ['console'],
-            'propagate': False,
-        },
-        'raven': {
-            'level': 'DEBUG',
-            'handlers': ['console'],
-            'propagate': False,
-        },
-        'sentry.errors': {
-            'level': 'DEBUG',
-            'handlers': ['console'],
+            'handlers': ['stream-to-stderr'],
             'propagate': False,
         },
     },
-    'root': {
-        'level': 'WARNING',
-        'handlers': ['sentry'],
-    }
 }
 
 # shell_plus convenience utilities
@@ -608,6 +592,3 @@ def route_urgent_task(name, args, kwargs, options, task=None, **kw):
         'SHOW_TOOLBAR_CALLBACK': lambda _: True
     }
     ALLOWED_HOSTS.append('localhost')
-
-if DEBUG and os.environ.get('PROF', False):
-    MIDDLEWARE += ('api.middleware.ProfileMiddleware', )

requirements.txt

@@ -31,8 +31,8 @@ psycopg2==2.9.5  # LGPL with exceptions or ZPL
 python-dateutil==2.8.1  # Apache 2.0
 PyJWE==1.0.0 # Apache 2.0
 pyyaml==6.0  # MIT
-raven==6.10.0  # BSD
 requests==2.25.1  # Apache 2.0
+sentry-sdk[django]==1.22.2  # MIT
 stevedore==3.3.0  # Apache 2.0
 xmltodict==0.12.0  # MIT