Merge pull request #97 from romgrk/experiments-service

Experiments service

chord_metadata_service/experiments/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class ExperimentsConfig(AppConfig):
+    name = 'chord_metadata_service.experiments'

chord_metadata_service/experiments/descriptions.py

@@ -0,0 +1,20 @@
+from chord_metadata_service.restapi.description_utils import EXTRA_PROPERTIES
+
+EXPERIMENT = {
+    "description": "A subject of a phenopacket, representing either a human (typically) or another organism.",
+    "properties": {
+        "id": "An arbitrary identifier for the experiment.",
+
+        "reference_registry_id": "The IHEC EpiRR ID for this dataset, only for IHEC Reference Epigenome datasets. Otherwise leave empty.",
+        "qc_flags": "Any quanlity control observations can be noted here. This field can be omitted if empty",
+        "experiment_type": "(Controlled Vocabulary) The assay target (e.g. ‘DNA Methylation’, ‘mRNA-Seq’, ‘smRNA-Seq’, 'Histone H3K4me1').",
+        "experiment_ontology": "(Ontology: OBI) links to experiment ontology information.",
+        "molecule_ontology": "(Ontology: SO) links to molecule ontology information.",
+        "molecule": "(Controlled Vocabulary) The type of molecule that was extracted from the biological material. Include one of the following: total RNA, polyA RNA, cytoplasmic RNA, nuclear RNA, small RNA, genomic DNA, protein, or other.",
+        "library_strategy": "(Controlled Vocabulary) The assay used. These are defined within the SRA metadata specifications with a controlled vocabulary (e.g. ‘Bisulfite-Seq’, ‘RNA-Seq’, ‘ChIP-Seq’). For a complete list, see https://www.ebi.ac.uk/ena/submit/reads-library-strategy.",
+
+        "other_fields": "The other fields for the experiment",
+
+        **EXTRA_PROPERTIES
+    }
+}

chord_metadata_service/experiments/migrations/0001_initial.py

@@ -0,0 +1,33 @@
+# Generated by Django 2.2.11 on 2020-03-25 19:50
+
+import chord_metadata_service.restapi.models
+import chord_metadata_service.restapi.validators
+import django.contrib.postgres.fields
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Experiment',
+            fields=[
+                ('id', models.CharField(help_text='An arbitrary identifier for the experiment.', max_length=200, primary_key=True, serialize=False)),
+                ('reference_registry_id', models.CharField(blank=True, help_text='The IHEC EpiRR ID for this dataset, only for IHEC Reference Epigenome datasets. Otherwise leave empty.', max_length=30, null=True)),
+                ('qc_flags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(help_text='Any quanlity control observations can be noted here. This field can be omitted if empty', max_length=100), default=list, null=True, size=None)),
+                ('experiment_type', models.CharField(help_text="(Controlled Vocabulary) The assay target (e.g. ‘DNA Methylation’, ‘mRNA-Seq’, ‘smRNA-Seq’, 'Histone H3K4me1').", max_length=30)),
+                ('experiment_ontology', django.contrib.postgres.fields.jsonb.JSONField(blank=True, help_text='(Ontology: OBI) links to experiment ontology information.', null=True, validators=[chord_metadata_service.restapi.validators.JsonSchemaValidator({'$id': 'ONTOLOGY_CLASS_LIST', '$schema': 'http://json-schema.org/draft-07/schema#', 'description': 'Ontology class list', 'items': {'$id': 'ONTOLOGY_CLASS', '$schema': 'http://json-schema.org/draft-07/schema#', 'additionalProperties': False, 'description': 'todo', 'properties': {'id': {'description': 'CURIE style identifier.', 'type': 'string'}, 'label': {'description': 'Human-readable class name.', 'type': 'string'}}, 'required': ['id', 'label'], 'title': 'Ontology class schema', 'type': 'object'}, 'title': 'Ontology class list', 'type': 'array'})])),
+                ('molecule_ontology', django.contrib.postgres.fields.jsonb.JSONField(blank=True, help_text='(Ontology: SO) links to molecule ontology information.', null=True, validators=[chord_metadata_service.restapi.validators.JsonSchemaValidator({'$id': 'ONTOLOGY_CLASS_LIST', '$schema': 'http://json-schema.org/draft-07/schema#', 'description': 'Ontology class list', 'items': {'$id': 'ONTOLOGY_CLASS', '$schema': 'http://json-schema.org/draft-07/schema#', 'additionalProperties': False, 'description': 'todo', 'properties': {'id': {'description': 'CURIE style identifier.', 'type': 'string'}, 'label': {'description': 'Human-readable class name.', 'type': 'string'}}, 'required': ['id', 'label'], 'title': 'Ontology class schema', 'type': 'object'}, 'title': 'Ontology class list', 'type': 'array'})])),
+                ('molecule', models.CharField(blank=True, choices=[('total RNA', 'total RNA'), ('polyA RNA', 'polyA RNA'), ('cytoplasmic RNA', 'cytoplasmic RNA'), ('nuclear RNA', 'nuclear RNA'), ('small RNA', 'small RNA'), ('genomic DNA', 'genomic DNA'), ('protein', 'protein'), ('other', 'other')], help_text='(Controlled Vocabulary) The type of molecule that was extracted from the biological material. Include one of the following: total RNA, polyA RNA, cytoplasmic RNA, nuclear RNA, small RNA, genomic DNA, protein, or other.', max_length=20, null=True)),
+                ('library_strategy', models.CharField(choices=[('DNase-Hypersensitivity', 'DNase-Hypersensitivity'), ('ATAC-seq', 'ATAC-seq'), ('NOME-Seq', 'NOME-Seq'), ('Bisulfite-Seq', 'Bisulfite-Seq'), ('MeDIP-Seq', 'MeDIP-Seq'), ('MRE-Seq', 'MRE-Seq'), ('ChIP-Seq', 'ChIP-Seq'), ('RNA-Seq', 'RNA-Seq'), ('miRNA-Seq', 'miRNA-Seq'), ('WGS', 'WGS')], help_text='(Controlled Vocabulary) The assay used. These are defined within the SRA metadata specifications with a controlled vocabulary (e.g. ‘Bisulfite-Seq’, ‘RNA-Seq’, ‘ChIP-Seq’). For a complete list, see https://www.ebi.ac.uk/ena/submit/reads-library-strategy.', max_length=25)),
+                ('other_fields', django.contrib.postgres.fields.jsonb.JSONField(blank=True, help_text='The other fields for the experiment', null=True, validators=[chord_metadata_service.restapi.validators.JsonSchemaValidator({'$id': 'KEY_VALUE_OBJECT', '$schema': 'http://json-schema.org/draft-07/schema#', 'additionalProperties': False, 'description': 'The schema represents a key-value object.', 'patternProperties': {'^.*$': {'type': 'string'}}, 'title': 'Key-value object', 'type': 'object'})])),
+            ],
+            bases=(models.Model, chord_metadata_service.restapi.models.IndexableMixin),
+        ),
+    ]

chord_metadata_service/experiments/models.py

@@ -0,0 +1,56 @@
+from django.db import models
+from django.db.models import CharField
+from django.core.validators import RegexValidator
+from django.core.exceptions import ValidationError
+from django.contrib.postgres.fields import JSONField, ArrayField
+from chord_metadata_service.restapi.models import IndexableMixin
+from chord_metadata_service.restapi.description_utils import rec_help
+from chord_metadata_service.restapi.validators import JsonSchemaValidator
+from chord_metadata_service.restapi.schemas import ONTOLOGY_CLASS_LIST, KEY_VALUE_OBJECT
+import chord_metadata_service.experiments.descriptions as d
+
+ontologyListValidator = JsonSchemaValidator(ONTOLOGY_CLASS_LIST)
+keyValueValidator     = JsonSchemaValidator(KEY_VALUE_OBJECT)
+
+class Experiment(models.Model, IndexableMixin):
+    """ Class to store Experiment information """
+
+    LIBRARY_STRATEGY = (
+        ('DNase-Hypersensitivity', 'DNase-Hypersensitivity'),
+        ('ATAC-seq', 'ATAC-seq'),
+        ('NOME-Seq', 'NOME-Seq'),
+        ('Bisulfite-Seq', 'Bisulfite-Seq'),
+        ('MeDIP-Seq', 'MeDIP-Seq'),
+        ('MRE-Seq', 'MRE-Seq'),
+        ('ChIP-Seq', 'ChIP-Seq'),
+        ('RNA-Seq', 'RNA-Seq'),
+        ('miRNA-Seq', 'miRNA-Seq'),
+        ('WGS', 'WGS'),
+    )
+
+    MOLECULE = (
+        ('total RNA', 'total RNA'),
+        ('polyA RNA', 'polyA RNA'),
+        ('cytoplasmic RNA', 'cytoplasmic RNA'),
+        ('nuclear RNA', 'nuclear RNA'),
+        ('small RNA', 'small RNA'),
+        ('genomic DNA', 'genomic DNA'),
+        ('protein', 'protein'),
+        ('other', 'other'),
+    )
+
+    id = CharField(primary_key=True, max_length=200, help_text=rec_help(d.EXPERIMENT, 'id'))
+
+    reference_registry_id = CharField(max_length=30, null=True, blank=True, help_text=rec_help(d.EXPERIMENT, 'reference_registry_id'))
+    qc_flags = ArrayField(CharField(max_length=100, help_text=rec_help(d.EXPERIMENT, 'qc_flags')), null=True, default=list)
+    experiment_type = CharField(max_length=30, null=False, blank=False, help_text=rec_help(d.EXPERIMENT, 'experiment_type'))
+    experiment_ontology = JSONField(null=True, blank=True, validators=[ontologyListValidator], help_text=rec_help(d.EXPERIMENT, 'experiment_ontology'))
+    molecule_ontology = JSONField(null=True, blank=True, validators=[ontologyListValidator], help_text=rec_help(d.EXPERIMENT, 'molecule_ontology'))
+    molecule = CharField(choices=MOLECULE, max_length=20, null=True, blank=True, help_text=rec_help(d.EXPERIMENT, 'molecule'))
+
+    library_strategy = CharField(choices=LIBRARY_STRATEGY, max_length=25, null=False, blank=False, help_text=rec_help(d.EXPERIMENT, 'library_strategy'))
+
+    other_fields = JSONField(blank=True, null=True, validators=[keyValueValidator], help_text=rec_help(d.EXPERIMENT, 'other_fields'))
+
+    def __str__(self):
+        return str(self.id)

chord_metadata_service/experiments/tests/test_models.py

@@ -0,0 +1,47 @@
+from django.test import TestCase
+from rest_framework import serializers
+from ..models import Experiment
+
+
+class ExperimentTest(TestCase):
+    """ Test module for Experiment model """
+
+    def setUp(self):
+        Experiment.objects.create(
+            id='experiment:1',
+            reference_registry_id='',
+            qc_flags=['flag 1', 'flag 2'],
+            experiment_type='Chromatin Accessibility',
+            experiment_ontology=[{"id": "ontology:1", "label": "Ontology term 1"}],
+            molecule_ontology=[{"id": "ontology:1", "label": "Ontology term 1"}],
+            molecule='total RNA',
+            library_strategy='Bisulfite-Seq',
+            other_fields={"some_field": "value"}
+        )
+
+    def create(self, **kwargs):
+        e = Experiment(**kwargs)
+        e.full_clean()
+        e.save()
+
+    def test_validation(self):
+        self.assertRaises(serializers.ValidationError, self.create,
+                id='experiment:2',
+                library_strategy='Bisulfite-Seq',
+                experiment_type='Chromatin Accessibility',
+                experiment_ontology=["invalid_value"],
+        )
+
+        self.assertRaises(serializers.ValidationError, self.create,
+                id='experiment:2',
+                library_strategy='Bisulfite-Seq',
+                experiment_type='Chromatin Accessibility',
+                molecule_ontology=[{"id": "some_id"}],
+        )
+
+        self.assertRaises(serializers.ValidationError, self.create,
+                id='experiment:2',
+                library_strategy='Bisulfite-Seq',
+                experiment_type='Chromatin Accessibility',
+                other_fields={"some_field": "value", "invalid_value": 42}
+        )

chord_metadata_service/metadata/settings.py

@@ -62,6 +62,7 @@
     'django.contrib.staticfiles',
 
     'chord_metadata_service.chord',
+    'chord_metadata_service.experiments.apps.ExperimentsConfig',
     'chord_metadata_service.patients.apps.PatientsConfig',
     'chord_metadata_service.phenopackets.apps.PhenopacketsConfig',
     'chord_metadata_service.restapi',

chord_metadata_service/restapi/schemas.py

@@ -65,7 +65,7 @@
 
 ONTOLOGY_CLASS = {
     "$schema": "http://json-schema.org/draft-07/schema#",
-    "$id": "todo",
+    "$id": "ONTOLOGY_CLASS",
     "title": "Ontology class schema",
     "description": "todo",
     "type": "object",
@@ -77,6 +77,15 @@
     "required": ["id", "label"]
 }
 
+ONTOLOGY_CLASS_LIST = {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$id": "ONTOLOGY_CLASS_LIST",
+    "title": "Ontology class list",
+    "description": "Ontology class list",
+    "type": "array",
+    "items": ONTOLOGY_CLASS,
+}
+
 EXTERNAL_REFERENCE = {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "$id": "todo",
@@ -123,6 +132,18 @@
     "required": ["evidence_code"]
 }
 
+KEY_VALUE_OBJECT = {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$id": "KEY_VALUE_OBJECT",
+    "title": "Key-value object",
+    "description": "The schema represents a key-value object.",
+    "type": "object",
+    "patternProperties": {
+        "^.*$": { "type": "string" }
+    },
+    "additionalProperties": False
+}
+
 
 AGE = {"type": "string", "description": "An ISO8601 string represent age."}
 

chord_metadata_service/restapi/validators.py

@@ -1,15 +1,24 @@
 from rest_framework import serializers
 from jsonschema import Draft7Validator
 
-
 class JsonSchemaValidator(object):
     """ Custom class based validator to validate against Json schema for JSONField """
 
     def __init__(self, schema):
         self.schema = schema
+        self.validator = Draft7Validator(self.schema)
 
     def __call__(self, value):
-        validation = Draft7Validator(self.schema).is_valid(value)
-        if not validation:
+        if not self.validator.is_valid(value):
             raise serializers.ValidationError("Not valid JSON schema for this field.")
         return value
+
+    def __eq__(self, other):
+        return self.schema == other.schema
+
+    def deconstruct(self):
+        return (
+            'chord_metadata_service.restapi.validators.JsonSchemaValidator',
+            [self.schema],
+            {}
+        )