Created an endpoint selection modal popup for users to save, add improvements on endpoint selection

.github/workflows/frontend.yml

@@ -47,7 +47,7 @@ jobs:
         env:
           RELEASE: dev
           ENV_FILE: .envs/.react
-          HTML_PATH: public/
+          HTML_PATH: public
         run: |
           # Replaces react-scripts substitution during build for index.html and generates runtime_env.js
           docker/production/react/entrypoint

.gitignore

@@ -189,3 +189,4 @@ tags
 .env
 .envs/*
 !.envs/.local/
+frontend/public/runtime_env.js

backend/.envs/.django

@@ -38,7 +38,5 @@ DJANGO_LOGIN_REDIRECT_URL=http://localhost:3000/search
 DJANGO_LOGOUT_REDIRECT_URL=http://localhost:3000/search
 
 # https://app.globus.org/settings/developers/registration/confidential_client
-# GLOBUS_CLIENT_KEY=12345
-# GLOBUS_CLIENT_SECRET=12345
-GLOBUS_CLIENT_KEY=998c6dc4-2939-4714-9004-5ac5b804abea
-GLOBUS_CLIENT_SECRET=xGgpQ2mt4TOVijoGTPkp7GdQZQoB29EMzgOqNdQ6Dzw=
+GLOBUS_CLIENT_KEY=12345
+GLOBUS_CLIENT_SECRET=12345

backend/README.md

@@ -24,6 +24,9 @@ docker compose -p metagrid_backend_dev build --no-cache
 # Run pyTest (May need to rebuild container before tests)
 docker compose -p metagrid_backend_dev run --rm django pytest
 
+# Run pyTest for a specific test (example)
+docker compose -p metagrid_backend_dev run --rm django pytest metagrid/api_proxy/tests/test_views.py::TestProxyViewSet::test_do_globus_auth
+
 # Run manage.py function
 docker compose -p metagrid_backend_dev run --rm django python manage.py <function>
 

backend/config/settings/base.py

@@ -154,7 +154,7 @@
     {
         "BACKEND": "django.template.backends.django.DjangoTemplates",
         "DIRS": STATICFILES_DIRS,
-        "APP_DIRS": True,
+        "APP_DIRS": False,
         "OPTIONS": {
             "context_processors": [
                 "django.template.context_processors.debug",

backend/docker-compose.prod.yml

@@ -1,5 +1,3 @@
-version: "3"
-
 volumes:
   production_postgres_data: {}
   production_postgres_data_backups: {}

backend/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3"
-
 volumes:
   local_postgres_data: {}
   local_postgres_data_backups: {}

backend/metagrid/api_proxy/tests/test_views.py

@@ -23,6 +23,22 @@ def test_globus_auth_begin(self):
         )
         self.assertEqual(response.status_code, 302)
 
+    def test_do_globus_get_endpoint(self):
+        url = reverse("globus-get-endpoint")
+
+        data = {"endpoint_id": "0247816e-cc0d-4e03-a509-10903f6dde11"}
+        response = self.client.get(url, data)
+        print(response.status_code)
+        assert response.status_code == status.HTTP_200_OK
+
+    def test_do_globus_search_endpoints(self):
+        url = reverse("globus-search-endpoints")
+
+        data = {"search_text": "0247816e-cc0d-4e03-a509-10903f6dde11"}
+        response = self.client.get(url, data)
+        print(response.status_code)
+        assert response.status_code == status.HTTP_200_OK
+
     def test_globus_auth_complete(self):
         url = reverse("globus-auth")
         User = get_user_model()

backend/metagrid/api_proxy/views.py

@@ -44,48 +44,11 @@ def do_globus_auth(request):
     )
 
 
-@api_view()
-@permission_classes([])
-def do_globus_get_endpoint(request):
-    endpoint_id = request.GET.get("endpoint_id", None)
-    client = globus_sdk.ConfidentialAppAuthClient(
-        settings.SOCIAL_AUTH_GLOBUS_KEY, settings.SOCIAL_AUTH_GLOBUS_SECRET
-    )
-    token_response = client.oauth2_client_credentials_tokens()
-    globus_transfer_data = token_response.by_resource_server[
-        "transfer.api.globus.org"
-    ]
-    globus_transfer_token = globus_transfer_data["access_token"]
-    authorizer = globus_sdk.AccessTokenAuthorizer(globus_transfer_token)
-    tc = globus_sdk.TransferClient(authorizer=authorizer)
-    endpoint = tc.get_endpoint(endpoint_id)
-    return Response(endpoint.data)
-
-
-@api_view()
-@permission_classes([])
-def do_globus_search_endpoints(request):
-    search_text = request.GET.get("search_text", None)
-    client = globus_sdk.ConfidentialAppAuthClient(
-        settings.SOCIAL_AUTH_GLOBUS_KEY, settings.SOCIAL_AUTH_GLOBUS_SECRET
-    )
-    token_response = client.oauth2_client_credentials_tokens()
-    globus_transfer_data = token_response.by_resource_server[
-        "transfer.api.globus.org"
-    ]
-    globus_transfer_token = globus_transfer_data["access_token"]
-    authorizer = globus_sdk.AccessTokenAuthorizer(globus_transfer_token)
-    tc = globus_sdk.TransferClient(authorizer=authorizer)
-    endpoints = tc.endpoint_search(filter_fulltext=search_text)
-    return Response(endpoints["DATA"])
-
-
 @csrf_exempt
 def do_globus_logout(request):
     logout(request)
     homepage_url = getattr(
-        settings, "DJANGO_LOGOUT_REDIRECT_URL", "http://localhost:3000/search/"
-    )
+        settings, "LOGOUT_REDIRECT_URL")
     return redirect(homepage_url)
 
 
@@ -94,7 +57,7 @@ def do_globus_logout(request):
 def do_globus_get_endpoint(request):
     endpoint_id = request.GET.get("endpoint_id", None)
     if request.user.is_authenticated:
-        tc = load_transfer_client(request.user)
+        tc = load_transfer_client(request.user)  # pragma: no cover
     else:
         client = globus_sdk.ConfidentialAppAuthClient(
             settings.SOCIAL_AUTH_GLOBUS_KEY, settings.SOCIAL_AUTH_GLOBUS_SECRET
@@ -114,11 +77,13 @@ def do_globus_get_endpoint(request):
 @permission_classes([])
 def do_globus_search_endpoints(request):
     search_text = request.GET.get("search_text", None)
+
     if request.user.is_authenticated:
-        tc = load_transfer_client(request.user)
+        tc = load_transfer_client(request.user)  # pragma: no cover
     else:
         client = globus_sdk.ConfidentialAppAuthClient(
-            settings.SOCIAL_AUTH_GLOBUS_KEY, settings.SOCIAL_AUTH_GLOBUS_SECRET
+            settings.SOCIAL_AUTH_GLOBUS_KEY,
+            settings.SOCIAL_AUTH_GLOBUS_SECRET,
         )
         token_response = client.oauth2_client_credentials_tokens()
         globus_transfer_data = token_response.by_resource_server[

backend/requirements/base.txt

@@ -1,7 +1,7 @@
 # Core
 # ------------------------------------------------------------------------------
 pytz==2023.3  # https://github.com/stub42/pytz
-django==4.2.7  # https://www.djangoproject.com/
+django==4.2.10  # https://www.djangoproject.com/
 django-environ==0.10.0  # https://github.com/joke2k/django-environ
 gunicorn==20.1.0  # https://github.com/benoitc/gunicorn
 newrelic==8.8.1  # https://pypi.org/project/newrelic/

docs/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3"
-
 services:
   docs:
     build:

frontend/Makefile

@@ -1,15 +1,20 @@
+TAG ?= v1.1.0-beta
+IMAGE ?= ghcr.io/esgf2-us/metagrid-frontend:$(TAG)
+
 .PHONY: build
 build:
-	docker build $(ARGS) -t react:latest -f docker/production/react/Dockerfile .
+	docker build $(ARGS) -t $(IMAGE) -f docker/production/react/Dockerfile .
 
 .PHONY: build-local
 build-local:
-	docker build $(ARGS) -t react:latest -f docker/local/Dockerfile .
+	docker build $(ARGS) -t $(IMAGE) -f docker/local/Dockerfile .
 
 .PHONY: run
+run: ARGS ?= -e RELEASE=production
 run:
-	docker run $(ARGS) -it --rm -p 3000:3000 react:latest
+	docker run $(ARGS) -it --rm -p 3000:3000 $(IMAGE)
 
 .PHONY: shell
+run: ARGS ?= -e RELEASE=production
 shell:
-	docker run $(ARGS) -it --rm -p 3000:3000 --entrypoint /bin/sh react:latest
+	docker run $(ARGS) -it --rm -p 3000:3000 --entrypoint /bin/sh $(IMAGE)

frontend/docker-compose.prod.yml

@@ -1,5 +1,3 @@
-version: '3'
-
 services:
   react:
     build:

frontend/docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3'
-
 services:
   react:
     build:

frontend/docker/production/react/entrypoint

@@ -1,28 +1,30 @@
 #!/bin/sh
 
-[ -n "${DEBUG}" ] && set -x
+DEBUG="${DEBUG:-false}"
+
+[ "${DEBUG}" == "true" ] && set -x
+
+TMP_PATH=/tmp
 
 export RELEASE="${RELEASE:-production}"
 export ENV_FILE="${ENV_FILE:-/env}"
 export HTML_PATH="${HTML_PATH:-/usr/share/nginx/html}"
 export STATIC_PATH=""
-export STATIC_URL="${PUBLIC_URL}${STATIC_PATH}"
-export RUNTIME_PATH="${HTML_PATH}${STATIC_PATH}/runtime_env.js"
 
-TMP_PATH=/tmp
-
-if [ ! -e "$(dirname ${RUNTIME_PATH})" ]
-then
-    mkdir -p "$(dirname ${RUNTIME_PATH})"
+if [ "${RELEASE}" = "production" ]; then
+    export STATIC_PATH="/static/js"
 fi
 
-if [ "${RELEASE}" = "production" ]
-then
-    export STATIC_PATH="/static/js"
+export STATIC_URL="${PUBLIC_URL}${STATIC_PATH}"
+export RUNTIME_ENV_FILE="${HTML_PATH}${STATIC_PATH}/runtime_env.js"
+
+if [ ! -e "$(dirname ${RUNTIME_ENV_FILE})" ]; then
+    mkdir -p "$(dirname ${RUNTIME_ENV_FILE})"
 fi
 
+# Create $RUNTIME_ENV_FILE from contents of $ENV_FILE
 if [ -e "${ENV_FILE}" ]; then
-    echo "window.ENV = {" > "${RUNTIME_PATH}"
+    echo "window.ENV = {" > "${RUNTIME_ENV_FILE}"
 
     while read -r line; do
         [ -z "$(echo ${line} | grep -vE '^# |^$')" ] && continue
@@ -36,31 +38,36 @@ if [ -e "${ENV_FILE}" ]; then
             export "${varname}"="${varvalue}"
         fi
 
-        echo "  ${varname}: \"${varvalue}\"," >> "${RUNTIME_PATH}"
+        echo "  ${varname}: \"${varvalue}\"," >> "${RUNTIME_ENV_FILE}"
     done < ${ENV_FILE}
 
-    echo "};" >> "${RUNTIME_PATH}"
+    echo "};" >> "${RUNTIME_ENV_FILE}"
 fi
 
 if [ ! -e "/index.html" ]; then
     cp "${HTML_PATH}/index.html" "${TMP_PATH}/index.html"
 fi
 
-if [ "${RELEASE}" = "production" ]
-then
-    if [ -z "${PUBLIC_URL}" ]
-    then
+# Template nginx configs
+if [ "${RELEASE}" = "production" ] && [ "${SKIP_NGINX_CONF:-false}" != "true" ]; then
+    echo "Writing nginx config"
+
+    if [ -z "${PUBLIC_URL}" ]; then
         envsubst '${PREVIOUS_URL}' < /nginx.conf > /etc/nginx/conf.d/default.conf
         export PUBLIC_URL=""
     else
         envsubst '${PREVIOUS_URL},${PUBLIC_URL}' < /nginx.subdir.conf > /etc/nginx/conf.d/default.conf
-
-        # Fixes react-scripts static path e.g. /static/ -> $PUBLIC_URL/static/
-        sed -i"" "s/\"\/static\//\"\$PUBLIC_URL\/static\//g" "${TMP_PATH}/index.html"
     fi
-
 fi
 
+# Fixes react-scripts static path e.g. /static/ -> $PUBLIC_URL/static/
+sed -i"" "s/\"\/static\//\"\$PUBLIC_URL\/static\//g" "${TMP_PATH}/index.html"
+
 envsubst '$STATIC_URL,$PUBLIC_URL,$REACT_APP_GOOGLE_ANALYTICS_TRACKING_ID' < "${TMP_PATH}/index.html" > "${HTML_PATH}/index.html"
 
+if [ "${DEBUG}" == "true" ]; then
+    cat "${HTML_PATH}/index.html"
+    cat "/etc/nginx/conf.d/default.conf"
+fi
+
 exec "$@"

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "frontend",
-  "version": "1.1.0",
+  "version": "1.1.2-rc",
   "private": true,
   "scripts": {
     "build:local": "env-cmd -f .envs/.react react-scripts build",

frontend/public/changelog/v1.1.0.md

@@ -7,3 +7,6 @@ This update includes several improvements, additional features, bug fixes and en
 1. Added ability to select a managed endpoint and obtain required scopes and permissions to successfully perform a transfer.
 2. Added feature to allow users to copy a list of the available facet options when selecting facet filters. The list is copied as text to the user's clipboard and include the result count to match the frontend display.
 3. Updated the search page to no longer use a button to confirm selected project. CMIP6 project will be selected by default.
+4. Upgrade `antd` frontend library to `v5`
+5. Upgrade `django` backend library to `4.2.10`
+6. **Bugfix** - support larger Globus Transfer dataset counts.

frontend/public/changelog/v1.1.1-pre.md

@@ -0,0 +1,8 @@
+## Summary
+
+This update includes a quick bug fix to address issues related to saved searches.
+
+**Changes**
+
+1. Fixed bug related to saved search where the save action would fail if logged-in due to csrf tokens
+2. Fixed issue where clicking saved search wouldn't restore the saved search in the search page

frontend/public/changelog/v1.1.2-rc.md

@@ -0,0 +1,7 @@
+## Summary
+
+This update includes several improvements, bug fixes and enhancements.
+
+**Changes**
+
+1. Added globus endpoint pop-up that allows users to search and save collections for transfers

frontend/public/messages/metagrid_messages.md

@@ -1,12 +1,8 @@
-# Welcome to the Metagrid Major Release v1.1.0
+# Welcome to the Metagrid Major Release v1.1.2-RC
 
 To view the latest documentation and FAQ, please visit this page:
 [https://esgf.github.io/esgf-user-support/metagrid.html](https://esgf.github.io/esgf-user-support/metagrid.html)
 
-## Globus Auth
-
-We now support logins via Globus Auth at LLNL!
-
 ## Globus Transfers enabled
 
 This version of Metagrid supports the user of Globus to transfer ESGF datasets to your institutional or personal endpoint. The feature can be accessed at the bottom of the Data Cart page. At present only data published at LLNL is available for Globus Transfer via Metagrid. Other sites may continue to have data transferrable using the _legacy_ CoG interface.