upgrade to django 4.2, postgres 16 and python 3.11 (#880, #1074, #1157)

.github/workflows/build.yml

@@ -10,9 +10,10 @@ jobs:
           - '3.8'
           - '3.9'
           - '3.10'
+          - '3.11'
     services:
       postgres:
-        image: postgres:11
+        image: postgres:16
         env:
           POSTGRES_DB: sodar_core
           POSTGRES_USER: sodar_core
@@ -44,7 +45,7 @@ jobs:
         run: git fetch --prune --unshallow
       - name: Install project Python dependencies
         run: |
-          pip install "wheel>=0.38.4, <0.39"
+          pip install "wheel>=0.40.0, <0.41"
           pip install -r requirements/local.txt
           pip install -r requirements/test.txt
       - name: Download icons
@@ -58,13 +59,13 @@ jobs:
           coverage report
       - name: Check linting
         run: flake8 .
-        if: ${{ matrix.python-version == '3.8' }}
+        if: ${{ matrix.python-version == '3.11' }}
       - name: Check formatting
         run: make black arg=--check
-        if: ${{ matrix.python-version == '3.8' }}
+        if: ${{ matrix.python-version == '3.11' }}
       - name: Report coverage with Coveralls
         uses: coverallsapp/github-action@master
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           path-to-lcov: './coverage.lcov'
-        if: ${{ matrix.python-version == '3.8' }}
+        if: ${{ matrix.python-version == '3.11' }}

CHANGELOG.rst

@@ -5,6 +5,24 @@ Changelog for the **SODAR Core** Django app package. Loosely follows the
 `Keep a Changelog <http://keepachangelog.com/en/1.0.0/>`_ guidelines.
 
 
+Unreleased
+==========
+
+Added
+-----
+
+- **General**
+    - Python v3.11 support (#1157)
+
+Changed
+-------
+
+- **General**
+    - Upgrade to Django v4.2 (#880)
+    - Upgrade minimum PostgreSQL version to v12 (#1074)
+    - Upgrade to PostgreSQL v16 in CI (#1074)
+
+
 v0.13.4 (2024-02-16)
 ====================
 

config/settings/base.py

@@ -2,10 +2,10 @@
 Django settings for the SODAR Core Example Site project.
 
 For more information on this file, see
-https://docs.djangoproject.com/en/dev/topics/settings/
+https://docs.djangoproject.com/en/4.2/topics/settings/
 
 For the full list of settings and their values, see
-https://docs.djangoproject.com/en/3.2/ref/settings/
+https://docs.djangoproject.com/en/4.2/ref/settings/
 """
 import environ
 import os
@@ -65,7 +65,7 @@
     'dal',  # For user search combo box
     'dal_select2',
     'dj_iconify.apps.DjIconifyConfig',  # Iconify for SVG icons
-    'django_saml2_auth',  # SAML2 support
+    # 'django_saml2_auth',  # SAML2 support, temp disabled (see #597, #880)
 ]
 
 # Project apps
@@ -143,12 +143,12 @@
     for x in env.list('ADMINS', default=['Admin User:admin@example.com'])
 ]
 
-# See: https://docs.djangoproject.com/en/3.2/ref/settings/#managers
+# See: https://docs.djangoproject.com/en/4.2/ref/settings/#managers
 MANAGERS = ADMINS
 
 # DATABASE CONFIGURATION
 # ------------------------------------------------------------------------------
-# See: https://docs.djangoproject.com/en/3.2/ref/settings/#databases
+# See: https://docs.djangoproject.com/en/4.2/ref/settings/#databases
 # Uses django-environ to accept uri format
 # See: https://django-environ.readthedocs.io/en/latest/#supported-types
 DATABASES = {
@@ -171,24 +171,24 @@
 # In a Windows environment this must be set to your system time zone.
 TIME_ZONE = 'Europe/Berlin'
 
-# See: https://docs.djangoproject.com/en/3.2/ref/settings/#language-code
+# See: https://docs.djangoproject.com/en/4.2/ref/settings/#language-code
 LANGUAGE_CODE = 'en-us'
 
-# See: https://docs.djangoproject.com/en/3.2/ref/settings/#site-id
+# See: https://docs.djangoproject.com/en/4.2/ref/settings/#site-id
 SITE_ID = 1
 
-# See: https://docs.djangoproject.com/en/3.2/ref/settings/#use-i18n
+# See: https://docs.djangoproject.com/en/4.2/ref/settings/#use-i18n
 USE_I18N = False
 
-# See: https://docs.djangoproject.com/en/3.2/ref/settings/#use-l10n
+# See: https://docs.djangoproject.com/en/4.2/ref/settings/#use-l10n
 USE_L10N = True
 
-# See: https://docs.djangoproject.com/en/3.2/ref/settings/#use-tz
+# See: https://docs.djangoproject.com/en/4.2/ref/settings/#use-tz
 USE_TZ = True
 
 # TEMPLATE CONFIGURATION
 # ------------------------------------------------------------------------------
-# See: https://docs.djangoproject.com/en/3.2/ref/settings/#templates
+# See: https://docs.djangoproject.com/en/4.2/ref/settings/#templates
 TEMPLATES = [
     {
         'BACKEND': 'django.template.backends.django.DjangoTemplates',
@@ -428,7 +428,9 @@
 # ------------------------------------------------------------------------------
 
 
-ENABLE_SAML = env.bool('ENABLE_SAML', False)
+# SAML support temporarily disabled (see #597, #880)
+ENABLE_SAML = False  # env.bool('ENABLE_SAML', False)
+
 SAML2_AUTH = {
     # Required setting
     # Pysaml2 Saml client settings

config/settings/local_target.py

@@ -5,7 +5,7 @@
 
 # DATABASE CONFIGURATION
 # ------------------------------------------------------------------------------
-# See: https://docs.djangoproject.com/en/3.2/ref/settings/#databases
+# See: https://docs.djangoproject.com/en/4.2/ref/settings/#databases
 # Uses django-environ to accept uri format
 # See: https://django-environ.readthedocs.io/en/latest/#supported-types
 DATABASES['default']['NAME'] = 'sodar_core_target'

config/settings/local_target2.py

@@ -5,7 +5,7 @@
 
 # DATABASE CONFIGURATION
 # ------------------------------------------------------------------------------
-# See: https://docs.djangoproject.com/en/3.2/ref/settings/#databases
+# See: https://docs.djangoproject.com/en/4.2/ref/settings/#databases
 # Uses django-environ to accept uri format
 # See: https://django-environ.readthedocs.io/en/latest/#supported-types
 DATABASES['default']['NAME'] = 'sodar_core_target2'

config/urls.py

@@ -7,7 +7,8 @@
 from django.urls import path
 from django.views import defaults as default_views
 
-import django_saml2_auth.views
+# SAML support temporarily disabled (see #597, #880)
+# import django_saml2_auth.views
 
 # Projectroles dependency
 from projectroles.views import HomeView
@@ -54,21 +55,22 @@
     path('examples/project/', include('example_project_app.urls')),
     # Example site app URLs
     path('examples/site/', include('example_site_app.urls')),
+    # SAML support temporarily disabled (see #597, #880)
     # These are the SAML2 related URLs. You can change "^saml2_auth/" regex to
     # any path you want, like "^sso_auth/", "^sso_login/", etc. (required)
-    path('saml2_auth/', include('django_saml2_auth.urls')),
+    # path('saml2_auth/', include('django_saml2_auth.urls')),
     # The following line will replace the default user login with SAML2 (optional)
     # If you want to specific the after-login-redirect-URL, use parameter "?next=/the/path/you/want"
     # with this view.
-    path('sso/login/', django_saml2_auth.views.signin),
+    # path('sso/login/', django_saml2_auth.views.signin),
     # The following line will replace the admin login with SAML2 (optional)
     # If you want to specific the after-login-redirect-URL, use parameter "?next=/the/path/you/want"
     # with this view.
-    path('sso/admin/login/', django_saml2_auth.views.signin),
+    # path('sso/admin/login/', django_saml2_auth.views.signin),
     # The following line will replace the default user logout with the signout page (optional)
-    path('sso/logout/', django_saml2_auth.views.signout),
+    # path('sso/logout/', django_saml2_auth.views.signout),
     # The following line will replace the default admin user logout with the signout page (optional)
-    path('sso/admin/logout/', django_saml2_auth.views.signout),
+    # path('sso/admin/logout/', django_saml2_auth.views.signout),
 ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
 
 

docs/source/app_projectroles_settings.rst

@@ -478,6 +478,13 @@ This part of the setup is **optional**.
 SAML SSO Configuration (Optional)
 =================================
 
+.. danger::
+
+    In the current dev version of SODAR Core (v1.1.0-WIP), SAML support has been
+    temporarily disabled. The repository must be upgraded to a new SAML library
+    with support for Django v4.2+. This may also cause changes for configuring
+    SAML authentication.
+
 Optional Single Sign-On (SSO) authorization via SAML is also available. To
 enable this feature, set ``ENABLE_SAML=1`` in your environment. Configuring SAML
 for SSO requires proper configuration of the Keycloak SSO server and the SAML

docs/source/conf.py

@@ -27,9 +27,9 @@
 author = 'BIH Core Unit Bioinformatics'
 
 # The short X.Y version
-version = '0.13'
+version = '1.0'
 # The full version, including alpha/beta/rc tags
-release = '0.13.4'
+release = '1.0-WIP'
 
 
 # -- General configuration ---------------------------------------------------

docs/source/dev_core_install.rst

@@ -26,7 +26,8 @@ system of choice.
 System Dependencies
 ===================
 
-To get started, install the OS dependencies, PostgreSQL >=11 and Python >=3.8.
+To get started, install the OS dependencies, Python >=3.8 (3.11 recommended) and
+PostgreSQL >=12.
 
 .. code-block:: console
 

docs/source/dev_project_app.rst

@@ -53,7 +53,7 @@ to set up a fresh app generated in the standard way with
 
 It is also assumed that apps are more or less created according to best
 practices defined by `Two Scoops <https://www.twoscoopspress.com/>`_, with the
-use of `Class-Based Views <https://docs.djangoproject.com/en/3.2/topics/class-based-views//>`_
+use of `Class-Based Views <https://docs.djangoproject.com/en/4.2/topics/class-based-views/>`_
 being a requirement.
 
 
@@ -96,7 +96,7 @@ To provide a unique identifier for objects in the SODAR context, add a
 
     When updating an existing Django model with an existing database, the
     ``sodar_uuid`` field needs to be populated. See
-    `instructions in Django documentation <https://docs.djangoproject.com/en/3.2/howto/writing-migrations/#migrations-that-add-unique-fields>`_
+    `instructions in Django documentation <https://docs.djangoproject.com/en/4.2/howto/writing-migrations/#migrations-that-add-unique-fields>`_
     on how to create the required migrations.
 
 Model Example

docs/source/getting_started.rst

@@ -56,9 +56,9 @@ your Django site are listed below. For a complete requirement list, see the
 - Ubuntu (20.04 Focal recommended and supported) / CentOS 7
 - System library requirements (see the ``utility`` directory and/or your own
   Django project)
-- Python >=3.8 (**NOTE:** Python 3.7 no longer supported in SODAR Core v0.10.8+)
-- Django 3.2
-- PostgreSQL >=11 and psycopg2-binary
+- Python 3.8-3.11 (3.11 recommended)
+- Django 4.2
+- PostgreSQL >=12 and psycopg2-binary
 - Bootstrap 4.x
 - JQuery 3.3.x
 - Shepherd and Tether

docs/source/index.rst

@@ -79,8 +79,8 @@ Python Programming
   any.
 
 Django Development
-  For learning about Django, head over to the `excellent documentation of the
-  Django Project <https://docs.djangoproject.com/en/3.2/>`_.
+  For learning about Django, head over to the
+  `official Django documentation <https://docs.djangoproject.com/en/4.2/>`_.
 
 HTML / Javascript / CSS / Bootstrap 4
   Together with Django, SODAR Core provides a framework to plug in your own

docs/source/major_changes.rst

@@ -10,6 +10,53 @@ older SODAR Core version. For a complete list of changes in current and previous
 releases, see the :ref:`full changelog<changelog>`.
 
 
+v1.0.0 (WIP)
+************
+
+Release Highlights
+==================
+
+- Upgrade to Django v4.2 and Postgres v16
+- Add Python 3.11 support
+
+Breaking Changes
+================
+
+Django v4.2 Upgrade
+-------------------
+
+This release updates SODAR Core from Django v3.2 to v4.2. This is a breaking
+change which causes many updates and also potentially requires updating several
+dependencies.
+
+To upgrade, please update the Django requirement along with your site's other
+Python requirements to match ones in ``requirements/*.txt``. See
+`Django deprecation documentation <https://docs.djangoproject.com/en/dev/internals/deprecation/>`_
+for details about what has been deprecated in Django and which changes are
+mandatory.
+
+Common known issues:
+
+- Minimum version of PostgreSQL has been bumped to v12.
+- Replace ``django.utils.translation.ugettext_lazy`` imports with
+  ``gettext_lazy``.
+- Replace ``django.conf.urls.url`` imports with ``django.urls.re_path``.
+- Calls for ``related_managers`` for unsaved objects raises ``ValueError``. This
+  can be handled by e.g. calling ``model.save(commit=False)`` before trying to
+  access foreign key relations.
+
+System Prerequisites
+--------------------
+
+The minimum required PostgreSQL version has been bumped to v12. In CI we
+currently use PostgreSQL v16, but anything from v12 to above should work with
+this SODAR Core release. It is strongly recommended to make backups of your
+production databases before upgrading.
+
+Python v3.11 support has been officially added in this version. 3.11 is now also
+the recommended Python version to use.
+
+
 v0.13.4 (2024-02-16)
 ********************
 

example_project_app/urls.py

@@ -1,5 +1,4 @@
-from django.conf.urls import url
-from django.urls import path
+from django.urls import path, re_path
 
 from example_project_app import views, views_api
 
@@ -10,14 +9,14 @@
 # NOTE: If referring to a model from another app, notation is "app__model"
 
 urls = [
-    url(
-        regex=r'^(?P<project>[0-9a-f-]+)$',
+    re_path(
+        r'^(?P<project>[0-9a-f-]+)$',
         view=views.ExampleView.as_view(),
         name='example',
     ),
     # Example view with model from an external app
-    url(
-        regex=r'^ext/(?P<filesfolders__folder>[0-9a-f-]+)$',
+    re_path(
+        r'^ext/(?P<filesfolders__folder>[0-9a-f-]+)$',
         view=views.ExampleView.as_view(),
         name='example_ext_model',
     ),
@@ -36,8 +35,8 @@
 ]
 
 urls_api = [
-    url(
-        regex=r'^api/hello/(?P<project>[0-9a-f-]+)$',
+    re_path(
+        r'^api/hello/(?P<project>[0-9a-f-]+)$',
         view=views_api.HelloExampleProjectAPIView.as_view(),
         name='example_api_hello',
     )

projectroles/forms.py

@@ -697,8 +697,8 @@ def __init__(self, project=None, current_user=None, *args, **kwargs):
     def clean(self):
         """Function for custom form validation and cleanup"""
         self.instance_owner_as = (
-            self.instance.get_owner() if self.instance else None
-        )
+            self.instance.get_owner() if self.instance.pk else None
+        )  # Must check pk, get_owner() with unsaved model fails in Django v4+
         disable_categories = getattr(
             settings, 'PROJECTROLES_DISABLE_CATEGORIES', False
         )