Merge pull request #117 from c3g/cleanup

Refactor schemas and separate search schema components

chord_metadata_service/chord/tests/test_api_search.py

@@ -10,11 +10,12 @@
 
 from chord_metadata_service.phenopackets.tests.constants import *
 from chord_metadata_service.phenopackets.models import *
+from chord_metadata_service.phenopackets.search_schemas import PHENOPACKET_SEARCH_SCHEMA
 
 from chord_metadata_service.chord.tests.es_mocks import SEARCH_SUCCESS
 from .constants import *
 from ..models import *
-from ..views_search import PHENOPACKET_DATA_TYPE_ID, PHENOPACKET_SCHEMA, PHENOPACKET_METADATA_SCHEMA
+from ..views_search import PHENOPACKET_DATA_TYPE_ID, PHENOPACKET_METADATA_SCHEMA
 
 
 class DataTypeTest(APITestCase):
@@ -31,15 +32,15 @@ def test_data_type_detail(self):
         c = r.json()
         self.assertDictEqual(c, {
             "id": PHENOPACKET_DATA_TYPE_ID,
-            "schema": PHENOPACKET_SCHEMA,
+            "schema": PHENOPACKET_SEARCH_SCHEMA,
             "metadata_schema": PHENOPACKET_METADATA_SCHEMA
         })
 
     def test_data_type_schema(self):
         r = self.client.get(reverse("data-type-schema"))  # Only mounted with phenopacket right now
         self.assertEqual(r.status_code, status.HTTP_200_OK)
         c = r.json()
-        self.assertDictEqual(c, PHENOPACKET_SCHEMA)
+        self.assertDictEqual(c, PHENOPACKET_SEARCH_SCHEMA)
 
     def test_data_type_metadata_schema(self):
         r = self.client.get(reverse("data-type-metadata-schema"))  # Only mounted with phenopacket right now
@@ -60,7 +61,7 @@ def dataset_rep(dataset, created, updated):
                 "created": created,
                 "updated": updated
             },
-            "schema": PHENOPACKET_SCHEMA
+            "schema": PHENOPACKET_SEARCH_SCHEMA
         }
 
     @override_settings(AUTH_OVERRIDE=True)  # For permissions

chord_metadata_service/chord/tests/test_search.py

@@ -1,13 +1,14 @@
 from django.test import TestCase
 from jsonschema import Draft7Validator
 
-from ..views_search import PHENOPACKET_SCHEMA, PHENOPACKET_METADATA_SCHEMA
+from chord_metadata_service.phenopackets.search_schemas import PHENOPACKET_SEARCH_SCHEMA
+from ..views_search import PHENOPACKET_METADATA_SCHEMA
 
 
 class SchemaTest(TestCase):
     @staticmethod
     def test_phenopacket_schema():
-        Draft7Validator.check_schema(PHENOPACKET_SCHEMA)
+        Draft7Validator.check_schema(PHENOPACKET_SEARCH_SCHEMA)
 
     @staticmethod
     def test_phenopacket_metadata_schema():

chord_metadata_service/chord/views_search.py

@@ -9,13 +9,13 @@
 from rest_framework.permissions import AllowAny
 from rest_framework.response import Response
 
-from chord_lib.responses.errors import *
+from chord_lib.responses import errors
 from chord_lib.search import build_search_response, postgres
 from chord_metadata_service.metadata.settings import DEBUG
 from chord_metadata_service.patients.models import Individual
 from chord_metadata_service.phenopackets.api_views import PHENOPACKET_PREFETCH
 from chord_metadata_service.phenopackets.models import Phenopacket
-from chord_metadata_service.phenopackets.schemas import PHENOPACKET_SCHEMA
+from chord_metadata_service.phenopackets.search_schemas import PHENOPACKET_SEARCH_SCHEMA
 from chord_metadata_service.phenopackets.serializers import PhenopacketSerializer
 from chord_metadata_service.metadata.elastic import es
 
@@ -33,23 +33,23 @@
 @api_view(["GET"])
 @permission_classes([AllowAny])
 def data_type_list(_request):
-    return Response([{"id": PHENOPACKET_DATA_TYPE_ID, "schema": PHENOPACKET_SCHEMA}])
+    return Response([{"id": PHENOPACKET_DATA_TYPE_ID, "schema": PHENOPACKET_SEARCH_SCHEMA}])
 
 
 @api_view(["GET"])
 @permission_classes([AllowAny])
 def data_type_phenopacket(_request):
     return Response({
         "id": PHENOPACKET_DATA_TYPE_ID,
-        "schema": PHENOPACKET_SCHEMA,
+        "schema": PHENOPACKET_SEARCH_SCHEMA,
         "metadata_schema": PHENOPACKET_METADATA_SCHEMA
     })
 
 
 @api_view(["GET"])
 @permission_classes([AllowAny])
 def data_type_phenopacket_schema(_request):
-    return Response(PHENOPACKET_SCHEMA)
+    return Response(PHENOPACKET_SEARCH_SCHEMA)
 
 
 @api_view(["GET"])
@@ -63,7 +63,7 @@ def data_type_phenopacket_metadata_schema(_request):
 def table_list(request):
     data_types = request.query_params.getlist("data-type")
     if PHENOPACKET_DATA_TYPE_ID not in data_types:
-        return Response(bad_request_error(f"Missing or invalid data type (Specified: {data_types})"), status=400)
+        return Response(errors.bad_request_error(f"Missing or invalid data type (Specified: {data_types})"), status=400)
 
     return Response([{
         "id": d.identifier,
@@ -74,7 +74,7 @@ def table_list(request):
             "created": d.created.isoformat(),
             "updated": d.updated.isoformat()
         },
-        "schema": PHENOPACKET_SCHEMA
+        "schema": PHENOPACKET_SEARCH_SCHEMA
     } for d in Dataset.objects.all()])
 
 
@@ -89,7 +89,7 @@ def table_detail(request, table_id):  # pragma: no cover
     try:
         table = Dataset.objects.get(identifier=table_id)
     except Dataset.DoesNotExist:
-        return Response(not_found_error(f"Table with ID {table_id} not found"), status=404)
+        return Response(errors.not_found_error(f"Table with ID {table_id} not found"), status=404)
 
     if request.method == "DELETE":
         table.delete()
@@ -167,7 +167,7 @@ def count_individual(ind):
         })
 
     except Dataset.DoesNotExist:
-        return Response(not_found_error(f"Table with ID {table_id} not found"), status=404)
+        return Response(errors.not_found_error(f"Table with ID {table_id} not found"), status=404)
 
 
 # TODO: CHORD-standardized logging
@@ -202,21 +202,24 @@ def phenopacket_query_results(query, params):
 
 def search(request, internal_data=False):
     if "data_type" not in request.data:
-        return Response(bad_request_error("Missing data_type in request body"), status=400)
+        return Response(errors.bad_request_error("Missing data_type in request body"), status=400)
 
     if "query" not in request.data:
-        return Response(bad_request_error("Missing query in request body"), status=400)
+        return Response(errors.bad_request_error("Missing query in request body"), status=400)
 
     start = datetime.now()
 
     if request.data["data_type"] != PHENOPACKET_DATA_TYPE_ID:
-        return Response(bad_request_error(f"Missing or invalid data type (Specified: {request.data['data_type']})"),
-                        status=400)
+        return Response(
+            errors.bad_request_error(f"Missing or invalid data type (Specified: {request.data['data_type']})"),
+            status=400
+        )
 
     try:
-        compiled_query, params = postgres.search_query_to_psycopg2_sql(request.data["query"], PHENOPACKET_SCHEMA)
+        compiled_query, params = postgres.search_query_to_psycopg2_sql(request.data["query"],
+                                                                       PHENOPACKET_SEARCH_SCHEMA)
     except (SyntaxError, TypeError, ValueError) as e:
-        return Response(bad_request_error(f"Error compiling query (message: {str(e)})"), status=400)
+        return Response(errors.bad_request_error(f"Error compiling query (message: {str(e)})"), status=400)
 
     if not internal_data:
         datasets = Dataset.objects.filter(identifier__in=phenopacket_results(
@@ -254,7 +257,7 @@ def chord_private_search(request):
 
 
 def phenopacket_filter_results(subject_ids, htsfile_ids, disease_ids, biosample_ids,
-                               phenotypicfeature_ids, phenopacket_ids, prefetch=False):
+                               phenotypicfeature_ids, phenopacket_ids):
 
     query = Phenopacket.objects.get_queryset()
 
@@ -286,7 +289,7 @@ def fhir_search(request, internal_data=False):
     # TODO: not all that sure about the query format we'll want
     # keep it simple for now
     if "query" not in request.data:
-        return Response(bad_request_error("Missing query in request body"), status=400)
+        return Response(errors.bad_request_error("Missing query in request body"), status=400)
 
     query = request.data["query"]
     start = datetime.now()
@@ -353,16 +356,17 @@ def chord_table_search(request, table_id, internal=False):
 
     if request.data is None or "query" not in request.data:
         # TODO: Better error
-        return Response(bad_request_error("Missing query in request body"), status=400)
+        return Response(errors.bad_request_error("Missing query in request body"), status=400)
 
     # Check that dataset exists
     dataset = Dataset.objects.get(identifier=table_id)
 
     try:
-        compiled_query, params = postgres.search_query_to_psycopg2_sql(request.data["query"], PHENOPACKET_SCHEMA)
+        compiled_query, params = postgres.search_query_to_psycopg2_sql(request.data["query"],
+                                                                       PHENOPACKET_SEARCH_SCHEMA)
     except (SyntaxError, TypeError, ValueError) as e:
         print("[CHORD Metadata] Error encountered compiling query {}:\n    {}".format(request.data["query"], str(e)))
-        return Response(bad_request_error(f"Error compiling query (message: {str(e)})"), status=400)
+        return Response(errors.bad_request_error(f"Error compiling query (message: {str(e)})"), status=400)
 
     debug_log(f"Finished compiling query in {datetime.now() - start}")
 

chord_metadata_service/mcode/api_views.py

@@ -1,9 +1,8 @@
 from rest_framework import viewsets
 from rest_framework.settings import api_settings
+from .models import *
 from .serializers import *
-from chord_metadata_service.restapi.api_renderers import (
-    PhenopacketsRenderer
-)
+from chord_metadata_service.restapi.api_renderers import PhenopacketsRenderer
 from chord_metadata_service.restapi.pagination import LargeResultsSetPagination
 
 

chord_metadata_service/mcode/descriptions.py

@@ -1,5 +1,6 @@
 # Most parts of this text are taken from the mCODE:Minimal Common Oncology Data Elements Data Dictionary.
-# The mCODE is made available under the Creative Commons 0 "No Rights Reserved" license https://creativecommons.org/share-your-work/public-domain/cc0/
+# The mCODE is made available under the Creative Commons 0 "No Rights Reserved" license
+# https://creativecommons.org/share-your-work/public-domain/cc0/
 
 # Portions of this text copyright (c) 2019-2020 the Canadian Centre for Computational Genomics; licensed under the
 # GNU Lesser General Public License version 3.

chord_metadata_service/mcode/models.py

@@ -66,8 +66,8 @@ class GeneticVariantFound(models.Model, IndexableMixin):
                                          help_text=rec_help(d.GENETIC_VARIANT_FOUND, "variant_found_identifier"))
     variant_found_hgvs_name = ArrayField(models.CharField(max_length=200), blank=True, null=True,
                                          help_text=rec_help(d.GENETIC_VARIANT_FOUND, "variant_found_hgvs_name"))
-    variant_found_description = models.CharField(max_length=200, blank=True,
-                                            help_text=rec_help(d.GENETIC_VARIANT_FOUND, "variant_found_description"))
+    variant_found_description = models.CharField(
+        max_length=200, blank=True, help_text=rec_help(d.GENETIC_VARIANT_FOUND, "variant_found_description"))
     # loinc value set https://loinc.org/48002-0/
     genomic_source_class = JSONField(blank=True, null=True, validators=[ontology_validator],
                                      help_text=rec_help(d.GENETIC_VARIANT_FOUND, "genomic_source_class"))
@@ -95,8 +95,8 @@ class GenomicsReport(models.Model, IndexableMixin):
 
     id = models.CharField(primary_key=True, max_length=200, help_text=rec_help(d.GENOMICS_REPORT, "id"))
     test_name = JSONField(validators=[ontology_validator], help_text=rec_help(d.GENOMICS_REPORT, "test_name"))
-    performing_organization_name = models.CharField(max_length=200, blank=True,
-                                                help_text=rec_help(d.GENOMICS_REPORT, "performing_organization_name"))
+    performing_organization_name = models.CharField(
+        max_length=200, blank=True, help_text=rec_help(d.GENOMICS_REPORT, "performing_organization_name"))
     specimen_type = JSONField(blank=True, null=True, validators=[ontology_validator],
                               help_text=rec_help(d.GENOMICS_REPORT, "specimen_type"))
     genetic_variant_tested = models.ManyToManyField(GeneticVariantTested, blank=True,
@@ -138,7 +138,7 @@ class LabsVital(models.Model, IndexableMixin):
                                          help_text=rec_help(d.LABS_VITAL, "blood_pressure_diastolic"))
     blood_pressure_systolic = JSONField(blank=True, null=True, validators=[quantity_validator],
                                         help_text=rec_help(d.LABS_VITAL, "blood_pressure_systolic"))
-    #TODO Change CodeableConcept to Ontology class
+    # TODO Change CodeableConcept to Ontology class
     tumor_marker_test = JSONField(validators=[tumor_marker_test_validator],
                                   help_text=rec_help(d.LABS_VITAL, "tumor_marker_test"))
     extra_properties = JSONField(blank=True, null=True,
@@ -247,7 +247,7 @@ class CancerRelatedProcedure(models.Model, IndexableMixin):
     occurence_time_or_period = JSONField(validators=[time_or_period_validator],
                                          help_text=rec_help(d.CANCER_RELATED_PROCEDURE, "occurence_time_or_period"))
     target_body_site = JSONField(null=True, validators=[ontology_list_validator],
-                                   help_text=rec_help(d.CANCER_RELATED_PROCEDURE, 'target_body_site'))
+                                 help_text=rec_help(d.CANCER_RELATED_PROCEDURE, 'target_body_site'))
     treatment_intent = JSONField(blank=True, null=True, validators=[ontology_validator],
                                  help_text=rec_help(d.CANCER_RELATED_PROCEDURE, "treatment_intent"))
     extra_properties = JSONField(blank=True, null=True,
@@ -261,8 +261,10 @@ class Meta:
     def __str__(self):
         return str(self.id)
 
+
 ###### Medication Statement ######
 
+
 class MedicationStatement(models.Model, IndexableMixin):
     """
     Class to record the use of a medication.

chord_metadata_service/mcode/serializers.py

@@ -3,6 +3,19 @@
 from .models import *
 
 
+__all__ = [
+    "GeneticVariantTestedSerializer",
+    "GeneticVariantFoundSerializer",
+    "GenomicsReportSerializer",
+    "LabsVitalSerializer",
+    "TNMStagingSerializer",
+    "CancerConditionSerializer",
+    "CancerRelatedProcedureSerializer",
+    "MedicationStatementSerializer",
+    "MCodePacketSerializer",
+]
+
+
 class GeneticVariantTestedSerializer(GenericSerializer):
 
     class Meta:

chord_metadata_service/patients/api_views.py

@@ -3,10 +3,7 @@
 from .serializers import IndividualSerializer
 from .models import Individual
 from chord_metadata_service.phenopackets.api_views import BIOSAMPLE_PREFETCH, PHENOPACKET_PREFETCH
-from chord_metadata_service.restapi.api_renderers import (
-    FHIRRenderer,
-    PhenopacketsRenderer
-)
+from chord_metadata_service.restapi.api_renderers import FHIRRenderer, PhenopacketsRenderer
 from chord_metadata_service.restapi.pagination import LargeResultsSetPagination
 
 
@@ -25,4 +22,4 @@ class IndividualViewSet(viewsets.ModelViewSet):
     ).order_by("id")
     serializer_class = IndividualSerializer
     pagination_class = LargeResultsSetPagination
-    renderer_classes = tuple(api_settings.DEFAULT_RENDERER_CLASSES) + (FHIRRenderer, PhenopacketsRenderer)
+    renderer_classes = (*api_settings.DEFAULT_RENDERER_CLASSES, FHIRRenderer, PhenopacketsRenderer)

chord_metadata_service/patients/models.py

@@ -2,7 +2,9 @@
 from django.contrib.postgres.fields import JSONField, ArrayField
 from chord_metadata_service.restapi.models import IndexableMixin
 from chord_metadata_service.restapi.validators import (
-    ontology_validator, age_or_age_range_validator, comorbid_condition_validator
+    ontology_validator,
+    age_or_age_range_validator,
+    comorbid_condition_validator,
 )
 
 
@@ -48,11 +50,12 @@ class Individual(models.Model, IndexableMixin):
     active = models.BooleanField(default=False, help_text='Whether this patient\'s record is in active use.')
     deceased = models.BooleanField(default=False, help_text='Indicates if the individual is deceased or not.')
     # mCode specific
-    # this field should be complex Ontology - clinical status and code - two Codeable concept - single, cl status has enum list of values
+    # this field should be complex Ontology - clinical status and code - two Codeable concept - single, cl status has
+    # enum list of values
     # TODO add these fields to FHIR converter ?
     comorbid_condition = JSONField(blank=True, null=True, validators=[comorbid_condition_validator],
                                    help_text='One or more conditions that occur with primary condition.')
-    #TODO decide use ONTOLOGY_CLASS vs. CODEABLE_CONCEPT - currently Ontology class
+    # TODO decide use ONTOLOGY_CLASS vs. CODEABLE_CONCEPT - currently Ontology class
     ecog_performance_status = JSONField(blank=True, null=True, validators=[ontology_validator],
                                         help_text='Value representing the Eastern Cooperative '
                                                   'Oncology Group performance status.')

chord_metadata_service/patients/serializers.py

@@ -1,18 +1,12 @@
-from chord_metadata_service.phenopackets.serializers import (
-    BiosampleSerializer,
-    SimplePhenopacketSerializer
-)
+from chord_metadata_service.phenopackets.serializers import BiosampleSerializer, SimplePhenopacketSerializer
 from chord_metadata_service.restapi.serializers import GenericSerializer
 from chord_metadata_service.restapi.fhir_utils import fhir_patient
 from .models import Individual
 
 
 class IndividualSerializer(GenericSerializer):
-    biosamples = BiosampleSerializer(
-        read_only=True, many=True, exclude_when_nested=['individual'])
-
-    phenopackets = SimplePhenopacketSerializer(
-        read_only=True, many=True, exclude_when_nested=['subject'])
+    biosamples = BiosampleSerializer(read_only=True, many=True, exclude_when_nested=['individual'])
+    phenopackets = SimplePhenopacketSerializer(read_only=True, many=True, exclude_when_nested=['subject'])
 
     class Meta:
         model = Individual