Updates to various packages and tests

.github/workflows/backend.yml

@@ -12,15 +12,20 @@ defaults:
 env:
   DJANGO_SETTINGS_MODULE: metagrid.config.local
   DOMAIN_NAME: http://localhost:8000
+  DJANGO_SECURE_SSL_REDIRECT: False
   CORS_ORIGIN_WHITELIST: http://localhost:3000
-  KEYCLOAK_URL: http://keycloak:8080/auth
-  KEYCLOAK_REALM: metagrid
-  KEYCLOAK_CLIENT_ID: backend
+  KEYCLOAK_URL: https://esgf-login.ceda.ac.uk/
+  KEYCLOAK_REALM: esgf
+  KEYCLOAK_CLIENT_ID: metagrid-localhost
   DATABASE_URL: pgsql://postgres:postgres@localhost:5432/postgres
   REACT_APP_SEARCH_URL: https://esgf-node.llnl.gov/esg-search/search
   REACT_APP_WGET_API_URL: https://esgf-node.llnl.gov/esg-search/wget
   REACT_APP_ESGF_NODE_STATUS_URL: https://aims2.llnl.gov/metagrid-backend/proxy/status
   REACT_APP_ESGF_SOLR_URL: https://esgf-fedtest.llnl.gov/solr
+  DJANGO_LOGIN_REDIRECT_URL: http://localhost:3000/search
+  DJANGO_LOGOUT_REDIRECT_URL: http://localhost:3000/search
+  GLOBUS_CLIENT_KEY: 12345
+  GLOBUS_CLIENT_SECRET: 12345
 
 jobs:
   build:

.github/workflows/frontend.yml

@@ -13,21 +13,20 @@ defaults:
     working-directory: frontend
 
 env:
-  REACT_APP_CORS_PROXY_URL: http://localhost:3001
-  REACT_APP_KEYCLOAK_REALM: metagrid
-  REACT_APP_KEYCLOAK_URL: http://keycloak:8080/auth
-  REACT_APP_KEYCLOAK_CLIENT_ID: frontend
+  REACT_APP_KEYCLOAK_REALM: esgf
+  REACT_APP_KEYCLOAK_URL: https://esgf-login.ceda.ac.uk/
+  REACT_APP_KEYCLOAK_CLIENT_ID: metagrid-localhost
 
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
 
-      - name: Use Node.js 18.x
+      - name: Use Node.js 21.x
         uses: actions/setup-node@v2
         with:
-          node-version: "18.x"
+          node-version: "21.x"
 
       - name: Cache node modules
         uses: actions/cache@v2

.github/workflows/pre-commit.yml

@@ -13,10 +13,10 @@ jobs:
         uses: actions/checkout@v2
 
       # Required to run the local ESLint hook
-      - name: Use Node.js 18.x
+      - name: Use Node.js 21.x
         uses: actions/setup-node@v2
         with:
-          node-version: "18.x"
+          node-version: "21.x"
 
       - name: Cache node modules
         uses: actions/cache@v2

.pre-commit-config.yaml

@@ -26,9 +26,10 @@ repos:
         args: ["--config=backend/pyproject.toml"]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.790
+    rev: v0.950
     hooks:
       - id: mypy
+        additional_dependencies: ["types-requests"]
 
   # Front-end
   # ------------------------------------------------------------------------------

.vscode/settings.json

@@ -59,6 +59,7 @@
     "typescript",
     "typescriptreact"
   ],
+  "eslint.workingDirectories": ["./frontend/"],
   // Jest
   // -----------------------------
   "jest.pathToJest": "yarn test:watch",
@@ -76,7 +77,5 @@
   "[jsonc]": {
     "editor.defaultFormatter": "esbenp.prettier-vscode"
   },
-  "python.analysis.extraPaths": [
-    "backend/venv/bin/python"
-  ]
+  "python.analysis.extraPaths": ["backend/venv/bin/python"]
 }

backend/.coveragerc

@@ -11,7 +11,8 @@ omit =
     *postgres-data/*,
     *tests/*,
     *venv/*,
-    *config/*
+    *config/*,
+    *tests*
 
 [report]
 # Regexes for lines to exclude from consideration

backend/.envs/.django

@@ -3,15 +3,19 @@
 DJANGO_SETTINGS_MODULE=config.settings.local
 DOMAIN_NAME=http://localhost:8000
 
+# Security
+# ------------------------------------------------------------------------------
+DJANGO_SECURE_SSL_REDIRECT=False
+
 # django-cors-headers
 # ------------------------------------------------------------------------------
 CORS_ORIGIN_WHITELIST=http://localhost:3000
 
 # django-all-auth
 # ------------------------------------------------------------------------------
-KEYCLOAK_URL=http://keycloak:8080/auth
-KEYCLOAK_REALM=metagrid
-KEYCLOAK_CLIENT_ID=backend
+KEYCLOAK_URL=https://esgf-login.ceda.ac.uk/
+KEYCLOAK_REALM=esgf
+KEYCLOAK_CLIENT_ID=metagrid-localhost
 
 # ESGF wget API
 # https://github.com/ESGF/esgf-wget
@@ -27,3 +31,12 @@ REACT_APP_ESGF_NODE_STATUS_URL=https://aims2.llnl.gov/metagrid-backend/proxy/sta
 
 # ESGF Solr URL
 REACT_APP_ESGF_SOLR_URL=https://esgf-fedtest.llnl.gov/solr
+
+# https://docs.djangoproject.com/en/4.2/ref/settings/#login-redirect-url
+# https://docs.djangoproject.com/en/4.2/ref/settings/#logout-redirect-url
+DJANGO_LOGIN_REDIRECT_URL=http://localhost:3000/search
+DJANGO_LOGOUT_REDIRECT_URL=http://localhost:3000/search
+
+# https://app.globus.org/settings/developers/registration/confidential_client
+GLOBUS_CLIENT_KEY=12345
+GLOBUS_CLIENT_SECRET=12345

backend/config/settings/base.py

@@ -62,7 +62,9 @@
     "allauth.socialaccount.providers.keycloak",
     "dj_rest_auth",
     "drf_yasg",
+    "social_django",
     # Your apps
+    "metagrid.api_proxy",
     "metagrid.users",
     "metagrid.projects",
     "metagrid.cart",
@@ -80,10 +82,18 @@
     "django.contrib.auth.middleware.AuthenticationMiddleware",
     "django.contrib.messages.middleware.MessageMiddleware",
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
+    "social_django.middleware.SocialAuthExceptionMiddleware",
 )
 
 SESSION_SAVE_EVERY_REQUEST = True
 
+# Authentication backends setup OAuth2 handling and where user data should be
+# stored
+AUTHENTICATION_BACKENDS = [
+    "globus_portal_framework.auth.GlobusOpenIdConnect",
+    "django.contrib.auth.backends.ModelBackend",
+]
+
 # https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts
 ALLOWED_HOSTS = ["localhost", "0.0.0.0", "127.0.0.1"]
 ROOT_URLCONF = "config.urls"
@@ -250,6 +260,8 @@
         "rest_framework.permissions.IsAuthenticated"
     ],
     "DEFAULT_AUTHENTICATION_CLASSES": (
+        "rest_framework.authentication.BasicAuthentication",
+        "rest_framework.authentication.SessionAuthentication",
         "dj_rest_auth.jwt_auth.JWTCookieAuthentication",
     ),
 }
@@ -287,6 +299,32 @@
 # Access tokens are used to validate a user
 ACCOUNT_EMAIL_VERIFICATION = "none"
 
+# social_django
+# -------------------------------------------------------------------------------
+# This is a general Django setting if views need to redirect to login
+# https://docs.djangoproject.com/en/3.2/ref/settings/#login-url
+LOGIN_URL = "/login/globus/"
+
+LOGIN_REDIRECT_URL = env("DJANGO_LOGIN_REDIRECT_URL")
+LOGOUT_REDIRECT_URL = env("DJANGO_LOGOUT_REDIRECT_URL")
+
+# This dictates which scopes will be requested on each user login
+SOCIAL_AUTH_GLOBUS_SCOPE = [
+    "openid",
+    "profile",
+    "email",
+    "urn:globus:auth:scope:search.api.globus.org:all",
+    "urn:globus:auth:scope:transfer.api.globus.org:all",
+]
+
+SOCIAL_AUTH_GLOBUS_KEY = env("GLOBUS_CLIENT_KEY")
+SOCIAL_AUTH_GLOBUS_SECRET = env("GLOBUS_CLIENT_SECRET")
+SOCIAL_AUTH_GLOBUS_AUTH_EXTRA_ARGUMENTS = {
+    "requested_scopes": SOCIAL_AUTH_GLOBUS_SCOPE,
+    "prompt": None,
+}
+SOCIAL_AUTH_JSONFIELD_ENABLED = True
+
 # dj-rest-auth
 # -------------------------------------------------------------------------------
 # https://dj-rest-auth.readthedocs.io/en/latest/index.html
@@ -296,7 +334,9 @@
 # django-cors-headers
 # -------------------------------------------------------------------------------
 # https://github.com/adamchainz/django-cors-headers#setup
-CORS_ORIGIN_ALLOW_ALL = False
+CORS_ALLOWED_ORIGINS = [
+    "http://localhost:3000",
+]
 CORS_ALLOW_CREDENTIALS = True
 CORS_ORIGIN_WHITELIST = env.list("CORS_ORIGIN_WHITELIST")
 

backend/config/settings/local.py

@@ -39,3 +39,9 @@
     "whitenoise.runserver_nostatic",
     "django_extensions",
 ] + INSTALLED_APPS  # noqa F405
+
+CORS_ORIGIN_ALLOW_ALL = True
+
+CSRF_TRUSTED_ORIGINS = ["http://localhost:3000"]
+
+SOCIAL_AUTH_REDIRECT_IS_HTTPS = False

backend/config/settings/production.py

@@ -57,3 +57,5 @@
 # ------------------------------------------------------------------------------
 # Django Admin URL regex.
 ADMIN_URL = env("DJANGO_ADMIN_URL")
+
+CORS_ORIGIN_ALLOW_ALL = False

backend/config/urls.py

@@ -15,6 +15,8 @@
 from metagrid.api_globus.views import do_globus_transfer, get_access_token
 from metagrid.api_proxy.views import (
     do_citation,
+    do_globus_auth,
+    do_globus_logout,
     do_search,
     do_status,
     do_wget,
@@ -50,6 +52,10 @@ class KeycloakLogin(SocialLoginView):
         r"^$",
         RedirectView.as_view(url=reverse_lazy("api-root"), permanent=False),
     ),
+    # social_auth
+    path("", include("social_django.urls", namespace="social")),
+    path("proxy/globus-logout/", do_globus_logout, name="globus-logout"),
+    path("proxy/globus-auth/", do_globus_auth, name="globus-auth"),
     # all-auth
     path("accounts/", include("allauth.urls"), name="socialaccount_signup"),
     # dj-rest-auth

backend/docker-compose.yml

@@ -22,22 +22,6 @@ services:
     ports:
       - "5433:5432"
 
-  keycloak:
-    # Source: https://github.com/keycloak/keycloak-containers/blob/master/docker-compose-examples/keycloak-postgres.yml
-    image: jboss/keycloak
-    container_name: keycloak
-    # https://github.com/jhipster/generator-jhipster/issues/7157#issuecomment-367813386
-    depends_on:
-      - postgres
-    volumes:
-      - ./docker/local/keycloak:/opt/jboss/keycloak/imports
-    env_file:
-      - ./.envs/.keycloak
-    ports:
-      - "8080:8080"
-    command: -Dkeycloak.import=/opt/jboss/keycloak/imports/realm-export.json
-      -Dkeycloak.migration.strategy=IGNORE_EXISTING
-
   django:
     build:
       context: .
@@ -46,7 +30,6 @@ services:
     container_name: django
     depends_on:
       - postgres
-      - keycloak
     volumes:
       - .:/app
     env_file:

backend/docker/local/django/Dockerfile

@@ -1,5 +1,6 @@
 FROM python:3.9-slim-buster
 
+ENV DATABASE_URL postgres://postgres:postgres@postgres:5432/postgres
 ENV PYTHONUNBUFFERED 1
 ENV PYTHONDONTWRITEBYTECODE 1