Container update

.github/workflows/backend.yml

@@ -17,18 +17,18 @@ env:
   KEYCLOAK_REALM: metagrid
   KEYCLOAK_CLIENT_ID: backend
   DATABASE_URL: pgsql://postgres:postgres@localhost:5432/postgres
-  REACT_APP_ESGF_NODE_URL: https://esgf-node.llnl.gov/esg-search/search
-  REACT_APP_WGET_API_URL: https://greyworm1-rh7.llnl.gov/wget
-  REACT_APP_ESGF_NODE_STATUS_URL: https://aims4.llnl.gov/prometheus/api/v1/query?query=probe_success%7Bjob%3D%22http_2xx%22%2C+target%3D~%22.%2Athredds.%2A%22%7D
-
+  REACT_APP_SEARCH_URL: https://esgf-node.llnl.gov/esg-search/search
+  REACT_APP_WGET_API_URL: https://esgf-node.llnl.gov/esg-search/wget
+  REACT_APP_ESGF_NODE_STATUS_URL: https://aims2.llnl.gov/metagrid-backend/proxy/status
+  REACT_APP_ESGF_SOLR_URL: https://esgf-fedtest.llnl.gov/solr
 
 jobs:
   build:
     runs-on: ubuntu-latest
 
     services:
       postgres:
-        image: postgres:11.6
+        image: postgres:12.6
         env:
           POSTGRES_PASSWORD: postgres
         ports:

.github/workflows/containers.yml

@@ -0,0 +1,48 @@
+name: Container release
+
+on:
+  push:
+    tags:
+      - v.*
+  pull_request:
+    branches:
+
+jobs:
+  frontend:
+    name: Build/release frontend container
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: docker/setup-buildx-action@v2
+      - uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: docker/build-push-action@v4
+        with:
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          context: frontend/
+          file: frontend/docker/production/react/Dockerfile
+          push: true
+          tags: ghcr.io/aims-group/metagrid-frontend:latest
+  backend:
+    name: Build/release backend container
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: docker/setup-buildx-action@v2
+      - uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: docker/build-push-action@v4
+        with:
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          context: backend/
+          file: backend/docker/production/django/Dockerfile
+          push: true
+          tags: ghcr.io/aims-group/metagrid-backend:latest

.gitignore

@@ -5,6 +5,7 @@
 .pytest_cache/
 
 # Config backups
+metagrid_configs/metagrid_config
 metagrid_configs/backups
 
 ### Linux template

backend/.envs/.django

@@ -15,12 +15,15 @@ KEYCLOAK_CLIENT_ID=backend
 
 # ESGF wget API
 # https://github.com/ESGF/esgf-wget
-REACT_APP_WGET_API_URL=https://nimbus3.llnl.gov/wget
+REACT_APP_WGET_API_URL=https://esgf-node.llnl.gov/esg-search/wget
 
 # ESGF Search API
 # https://esgf.github.io/esg-search/ESGF_Search_RESTful_API.html
-REACT_APP_ESGF_NODE_URL=https://esgf-node.llnl.gov/esg-search/search
+REACT_APP_SEARCH_URL=https://esgf-node.llnl.gov/esg-search/search
 
 # ESGF Node Status API
 # https://github.com/ESGF/esgf-utils/blob/master/node_status/query_prom.py
 REACT_APP_ESGF_NODE_STATUS_URL=https://aims2.llnl.gov/metagrid-backend/proxy/status
+
+# ESGF Solr URL
+REACT_APP_ESGF_SOLR_URL=https://esgf-fedtest.llnl.gov/solr

backend/UpdateProjectData_README.txt

@@ -0,0 +1,45 @@
+STEPS TO UPDATE PROJECTS, FACETS OR CATEGORIES
+
+1. Edit the initial data file with the desired changes: metagrid/backend/metagrid/initial_projects_data.py
+2. Change to the backend directory:
+cd metagrid/backend/
+3. Make sure the docker traefik and backend containers are up and running.
+If not running, you can run the containers by going to the traefik directory first and running this command:
+sudo docker compose -f docker-compose.prod.yml up --build -d
+Then do the same in the backend directory (production backend depends on traefik)
+
+RUN UPDATE AND CLEAR TABLES
+If you need to clear tables to remove existing facets/projects or change their order then do the steps below:
+
+Option 1:
+4. Use the updateProjects.sh script. Just run the script using clear option:
+sudo ./updateProject.sh --clear
+
+DONE!
+
+Option 2: Manually update without the script and clear tables:
+4. sudo docker compose -f docker-compose.prod.yml build django # Build the container
+5. sudo docker compose -f docker-compose.prod.yml run --rm django python manage.py migrate projects zero
+6. sudo docker compose -f docker-compose.prod.yml run --rm django python manage.py migrate projects
+
+DONE!
+
+RUN UPDATE WITHOUT CLEARING TABLES
+If your update is small and only involves minor modifications or additions to existing projects then you don't need to clear the tables.
+If there are deletions or more significant changes, run steps above which include clearing tables otherwise do steps below:
+
+Option 1:
+4. Use the updateProjects.sh script. Just run the default script:
+sudo ./updateProject.sh
+
+DONE!
+
+Option 2: Manually update without the script and don't clear tables:
+4. sudo docker compose -f docker-compose.prod.yml build django # Build the container
+5. sudo docker compose -f docker-compose.prod.yml run --rm django python manage.py migrate --fake projects 0001_initial
+6. sudo docker compose -f docker-compose.prod.yml run --rm django python manage.py migrate projects
+
+DONE!
+
+MIGRATION FILE ISSUES:
+If there are issues when attempting the steps above, make sure you don't have obsolete or modified migration files that don't match what's in the latest repository.

backend/config/settings/base.py

@@ -82,6 +82,8 @@
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
 )
 
+SESSION_SAVE_EVERY_REQUEST = True
+
 # https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts
 ALLOWED_HOSTS = ["localhost", "0.0.0.0", "127.0.0.1"]
 ROOT_URLCONF = "config.urls"
@@ -272,7 +274,7 @@
         "KEYCLOAK_REALM": env(
             "KEYCLOAK_REALM",
         ),
-    }
+    },
 }
 # Used in data migration to register Keycloak social app
 KEYCLOAK_CLIENT_ID = env("KEYCLOAK_CLIENT_ID")
@@ -295,9 +297,10 @@
 # -------------------------------------------------------------------------------
 # https://github.com/adamchainz/django-cors-headers#setup
 CORS_ORIGIN_ALLOW_ALL = False
+CORS_ALLOW_CREDENTIALS = True
 CORS_ORIGIN_WHITELIST = env.list("CORS_ORIGIN_WHITELIST")
 
-
-SEARCH_URL = env("REACT_APP_ESGF_NODE_URL")
+SEARCH_URL = env("REACT_APP_SEARCH_URL")
 WGET_URL = env("REACT_APP_WGET_API_URL")
 STATUS_URL = env("REACT_APP_ESGF_NODE_STATUS_URL")
+SOLR_URL = env("REACT_APP_ESGF_SOLR_URL")

backend/config/urls.py

@@ -12,7 +12,15 @@
 from rest_framework import permissions
 from rest_framework.routers import DefaultRouter
 
-from metagrid.api_proxy.views import do_citation, do_search, do_status, do_wget
+from metagrid.api_globus.views import do_globus_transfer, get_access_token
+from metagrid.api_proxy.views import (
+    do_citation,
+    do_search,
+    do_status,
+    do_wget,
+    get_temp_storage,
+    set_temp_storage,
+)
 from metagrid.cart.views import CartViewSet, SearchViewSet
 from metagrid.projects.views import ProjectsViewSet
 from metagrid.users.views import UserCreateViewSet, UserViewSet
@@ -53,6 +61,10 @@ class KeycloakLogin(SocialLoginView):
     path(
         "dj-rest-auth/keycloak", KeycloakLogin.as_view(), name="keycloak_login"
     ),
+    path("tempStorage/get", get_temp_storage, name="temp_storage_get"),
+    path("tempStorage/set", set_temp_storage, name="temp_storage_set"),
+    path("globus/auth", get_access_token, name="globus_auth"),
+    path("globus/transfer", do_globus_transfer, name="globus_transfer"),
     re_path(
         r"^account-confirm-email/",
         VerifyEmailView.as_view(),

backend/docker/local/django/Dockerfile

@@ -16,7 +16,7 @@ RUN apt-get update \
 
 # Requirements are installed here to ensure they will be cached.
 COPY ./requirements /requirements
-RUN pip install -r /requirements/local.txt
+RUN pip3 install -r /requirements/local.txt
 
 COPY ./docker/production/django/entrypoint /entrypoint
 RUN sed -i 's/\r$//g' /entrypoint

backend/docker/production/django/Dockerfile

@@ -18,7 +18,7 @@ RUN addgroup --system django \
 
 # Requirements are installed here to ensure they will be cached.
 COPY ./requirements /requirements
-RUN pip install --no-cache-dir -r /requirements/production.txt \
+RUN pip3 install --no-cache-dir -r /requirements/production.txt \
     && rm -rf /requirements
 
 COPY ./docker/production/django/entrypoint /entrypoint

backend/docker/production/postgres/maintenance/backup

@@ -4,7 +4,7 @@
 ### Create a database backup.
 ###
 ### Usage:
-###     $ docker-compose -f <environment>.yml (exec |run --rm) postgres backup
+###     $ docker compose -f <environment>.yml (exec |run --rm) postgres backup
 
 
 set -o errexit

backend/docker/production/postgres/maintenance/restore

@@ -7,7 +7,7 @@
 ###     <1> filename of an existing backup.
 ###
 ### Usage:
-###     $ docker-compose -f <environment>.yml (exec |run --rm) postgres restore <1>
+###     $ docker compose -f <environment>.yml (exec |run --rm) postgres restore <1>
 
 
 set -o errexit

backend/metagrid/api_globus/tests/test_views.py

@@ -0,0 +1,56 @@
+from django.urls import reverse
+from rest_framework import status
+from rest_framework.test import APITestCase
+
+from metagrid.api_globus.views import split_value, truncate_urls
+
+
+class TestGlobusViewSet(APITestCase):
+    def test_truncate(self):
+        lst = [{"url": ["test_url:globus_value|Globus"]}]
+        results = []
+        for value in truncate_urls(lst):
+            results.append(value)
+        assert results == ["globus_value"]
+
+    def test_split_value(self):
+        result = split_value(1)
+        result2 = split_value(1.0)
+        result3 = split_value("noSplitting")
+        result4 = split_value("splitValues,values")
+        result5 = split_value("splitValues,values,CESM1(CAM5.1,FV2)")
+        result6 = split_value(
+            "{splitValues,test},[test,test2],CESM1(CAM5.1,FV2)"
+        )
+
+        assert result == 1
+        assert result2 == 1.0
+        assert result3 == ["noSplitting"]
+        assert result4 == ["splitValues", "values"]
+        assert result5 == ["splitValues", "values", "CESM1(CAM5.1,FV2)"]
+        assert result6 == [
+            "{splitValues,test}",
+            "[test,test2]",
+            "CESM1(CAM5.1,FV2)",
+        ]
+
+    def test_get_access_token(self):
+        url = reverse("globus_auth")
+        getdata = {}
+        response = self.client.post(url, getdata)
+        assert response.status_code == status.HTTP_404_NOT_FOUND
+
+    def test_globus_transfer(self):
+        url = reverse("globus_transfer")
+        getdata = {}
+        response = self.client.get(url, getdata)
+        assert response.status_code == status.HTTP_400_BAD_REQUEST
+
+        postdata = {
+            "access_token": "",
+            "refresh_token": "",
+            "endpointId": "test",
+            "path": "bad/path",
+        }
+        response = self.client.post(url, postdata)
+        assert response.status_code == status.HTTP_400_BAD_REQUEST