Merge pull request #340 from bento-platform/merger-candig

Merge CANdig's Katsu fork changes into main Katsu development

.env-sample

@@ -16,6 +16,10 @@ export CHORD_PERMISSIONS=
 export SERVICE_ID=
 
 # CanDIG-specific
+export INSIDE_CANDIG=true
 export CANDIG_AUTHORIZATION=OPA
 export CANDIG_OPA_URL=http://0.0.0.0:8181
 export CACHE_TIME=0
+export ROOT_CA=
+export CANDIG_OPA_VERSION=
+export PERMISSIONS_SECRET=

.github/workflows/docker-publish.yml

@@ -0,0 +1,31 @@
+# name: Docker
+
+# on:
+#   push:
+#     branches:
+#       - stable
+
+#     # Publish `v1.2.3` tags as releases.
+#     tags:
+#       - v*
+
+# jobs:
+  # push_to_registry:
+  #   name: Push Docker image to GitHub Container Registry
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Check out the repo
+  #       uses: actions/checkout@v2
+  #       with:
+  #         submodules: recursive
+  #         ref: stable
+  #     - name: Push to GitHub Packages
+  #       uses: docker/build-push-action@v1
+  #       with:
+  #         username: ${{ github.actor }}
+  #         password: ${{ github.token }}
+  #         registry: ghcr.io
+  #         repository: candig/candig-katsu
+  #         tag_with_ref: true
+  #         tags: latest
+  #         dockerfile: Dockerfile

.github/workflows/test.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.8, 3.9, 3.10.7]
     services:
       postgres:
         image: postgres:12
@@ -37,7 +37,7 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
-      run: python -m pip install -r requirements.txt
+      run: python -m pip install -r requirements-dev.txt
     - name: Test
       run: |
         export POSTGRES_USER="postgres" && export POSTGRES_PASSWORD="postgres" && export POSTGRES_PORT=5432

.travis.yml

@@ -0,0 +1,18 @@
+dist: focal
+language: python
+python:
+  - "3.10"
+addons:
+  postgresql: "12"
+  apt:
+    packages:
+      - postgresql-12
+      - postgresql-contrib-12
+before_install:
+  - sudo -u postgres psql -U postgres -p 5433 -d postgres -c "alter user postgres with password 'hj38f3Ntr';"
+install:
+  - pip install -r requirements-dev.txt
+script:
+  - export POSTGRES_USER="postgres" && export POSTGRES_PASSWORD="hj38f3Ntr" && export POSTGRES_PORT=5433
+  - python3 -m coverage run ./manage.py test
+  - codecov

Dockerfile

@@ -1,6 +1,6 @@
-ARG venv_python=3.7
-
-FROM python:${venv_python}-alpine3.13
+ARG venv_python
+ARG alpine_version
+FROM python:${venv_python}-alpine${alpine_version}
 
 LABEL Maintainer="CanDIG Team"
 
@@ -35,7 +35,7 @@ RUN apk add --no-cache \
 RUN mkdir /app
 WORKDIR /app
 ADD ./requirements.txt /app
-RUN pip install -r requirements.txt
+RUN pip install -r requirements-dev.txt
 
 COPY . /app/chord_metadata_service
 

chord_metadata_service/chord/api_views.py

@@ -7,6 +7,11 @@
 from .models import Project, Dataset, TableOwnership, Table
 from .permissions import OverrideOrSuperUserOnly
 from .serializers import ProjectSerializer, DatasetSerializer, TableOwnershipSerializer, TableSerializer
+from .filters import AuthorizedDatasetFilter
+from django_filters.rest_framework import DjangoFilterBackend
+import logging
+
+logger = logging.getLogger(__name__)
 
 
 __all__ = ["ProjectViewSet", "DatasetViewSet", "TableOwnershipViewSet", "TableViewSet"]
@@ -49,9 +54,12 @@ class DatasetViewSet(CHORDPublicModelViewSet):
     Create a new dataset
     """
 
-    queryset = Dataset.objects.all().order_by("identifier")
+    filter_backends = [DjangoFilterBackend]
+    filterset_class = AuthorizedDatasetFilter
+
     serializer_class = DatasetSerializer
     renderer_classes = tuple(CHORDModelViewSet.renderer_classes) + (JSONLDDatasetRenderer, RDFDatasetRenderer,)
+    queryset = Dataset.objects.all().order_by("title")
 
 
 class TableOwnershipViewSet(CHORDPublicModelViewSet):

chord_metadata_service/chord/filters.py

@@ -0,0 +1,54 @@
+import django_filters
+import logging
+
+logger = logging.getLogger(__name__)
+
+# HELPERS
+
+
+def filter_datasets(qs, name, value):
+    """
+    Filters by datasets.
+    If value is None, returns all objects regardless of datasets.
+    Otherwise, return objects that are in the specified datasets.
+    """
+    if value:
+        lookup = "__".join([name, "in"])
+        return qs.filter(**{lookup: value.split(",")}).distinct()
+    else:
+        return qs
+
+
+# TODO   authorize_datasets(): remove the code == GRU filter, urgently.
+def authorize_datasets(qs, name, value):
+    """
+    Filter by authorized datasets.
+    If value is 'NO_DATASETS_AUTHORIZED', returns no objects.
+    Otherwise, returns objects that are in the specified datasets.
+    """
+    logger.warn(f"value is {value}")
+    if value == "NO_DATASETS_AUTHORIZED":
+        lookup = "__".join([name, "in"])
+        return qs.filter(**{lookup: []})
+    else:
+        lookup = "__".join([name, "in"])
+
+        # TODO  THE FILTER BELOW IS JANKY; NEEDS TO BE REMOVED.
+        #       It is only here for the ClinDIG 4.3 demo.
+        temp = qs.filter(**{lookup: value.split(",")}).distinct()\
+            .filter(data_use__consent_code__primary_category__code='GRU')
+        for t in temp:
+            logger.warn(str(t.data_use))
+
+        return temp
+
+
+class AuthorizedDatasetFilter(django_filters.rest_framework.FilterSet):
+    datasets = django_filters.CharFilter(
+        method=filter_datasets, field_name="table_ownership__dataset__title",
+        label="Datasets"
+    )
+    authorized_datasets = django_filters.CharFilter(
+        method=authorize_datasets, field_name="table_ownership__dataset__title",
+        label="Authorized datasets"
+    )

chord_metadata_service/chord/migrations/0004_v2_14_0.py

@@ -0,0 +1,108 @@
+# Generated by Django 4.1.1 on 2022-09-22 19:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('chord', '0003_v2_13_0'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='dataset',
+            name='acknowledges',
+            field=models.JSONField(blank=True, default=list, help_text='The grant(s) which funded the work reported by the dataset.'),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='alternate_identifiers',
+            field=models.JSONField(blank=True, default=list, help_text='Alternate identifiers for the dataset.'),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='citations',
+            field=models.JSONField(blank=True, default=list, help_text='The publication(s) that cite this dataset.'),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='creators',
+            field=models.JSONField(blank=True, default=list, help_text='The person(s) or organization(s) which contributed to the creation of the dataset.'),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='data_use',
+            field=models.JSONField(),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='dates',
+            field=models.JSONField(blank=True, default=list, help_text='Relevant dates for the datasets, a date must be added, e.g. creation date or last modification date should be added.'),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='dimensions',
+            field=models.JSONField(blank=True, default=list, help_text='The different dimensions (granular components) making up a dataset.'),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='distributions',
+            field=models.JSONField(blank=True, default=list, help_text='The distribution(s) by which datasets are made available (for example: mySQL dump).'),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='extra_properties',
+            field=models.JSONField(blank=True, help_text='Extra properties that do not fit in the previous specified attributes.', null=True),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='has_part',
+            field=models.ManyToManyField(blank=True, help_text="A Dataset that is a subset of this Dataset; Datasets declaring the 'hasPart' relationship are considered a collection of Datasets, the aggregation criteria could be included in the 'description' field.", to='chord.dataset'),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='keywords',
+            field=models.JSONField(blank=True, default=list, help_text='Tags associated with the dataset, which will help in its discovery.'),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='licenses',
+            field=models.JSONField(blank=True, default=list, help_text='The terms of use of the dataset.'),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='linked_field_sets',
+            field=models.JSONField(blank=True, default=list, help_text='Data type fields which are linked together.'),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='primary_publications',
+            field=models.JSONField(blank=True, default=list, help_text='The primary publication(s) associated with the dataset, usually describing how the dataset was produced.'),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='produced_by',
+            field=models.JSONField(blank=True, help_text='A study process which generated a given dataset, if any.', null=True),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='related_identifiers',
+            field=models.JSONField(blank=True, default=list, help_text='Related identifiers for the dataset.'),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='spatial_coverage',
+            field=models.JSONField(blank=True, default=list, help_text='The geographical extension and span covered by the dataset and its measured dimensions/variables.'),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='stored_in',
+            field=models.JSONField(blank=True, help_text='The data repository hosting the dataset.', null=True),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='types',
+            field=models.JSONField(blank=True, default=list, help_text='A term, ideally from a controlled terminology, identifying the dataset type or nature of the data, placing it in a typology.'),
+        ),
+    ]