From ec613e80f70fafc24e1ef84d893d9b616a3864c9 Mon Sep 17 00:00:00 2001 From: john-hill Date: Thu, 7 Apr 2022 18:03:23 -0700 Subject: [PATCH 1/3] first attempt to represent duo as json-schema --- .../JsonSchemaWorkerIntegrationTest.java | 101 ++++++++++++++++-- .../test/resources/schema/DUO/D0000001.json | 6 ++ .../test/resources/schema/DUO/D0000004.json | 12 +++ .../test/resources/schema/DUO/D0000006.json | 12 +++ .../test/resources/schema/DUO/D0000007.json | 33 ++++++ .../test/resources/schema/DUO/D0000011.json | 12 +++ .../test/resources/schema/DUO/D0000012.json | 31 ++++++ .../test/resources/schema/DUO/D0000015.json | 12 +++ .../test/resources/schema/DUO/D0000016.json | 12 +++ .../test/resources/schema/DUO/D0000018.json | 12 +++ .../test/resources/schema/DUO/D0000019.json | 12 +++ .../test/resources/schema/DUO/D0000020.json | 27 +++++ .../test/resources/schema/DUO/D0000021.json | 12 +++ .../test/resources/schema/DUO/D0000022.json | 27 +++++ .../test/resources/schema/DUO/D0000024.json | 28 +++++ .../test/resources/schema/DUO/D0000025.json | 27 +++++ .../test/resources/schema/DUO/D0000026.json | 12 +++ .../test/resources/schema/DUO/D0000027.json | 27 +++++ .../test/resources/schema/DUO/D0000028.json | 27 +++++ .../test/resources/schema/DUO/D0000029.json | 12 +++ .../test/resources/schema/DUO/D0000042.json | 12 +++ .../test/resources/schema/DUO/D0000043.json | 12 +++ .../test/resources/schema/DUO/D0000044.json | 12 +++ .../test/resources/schema/DUO/D0000045.json | 12 +++ .../test/resources/schema/DUO/D0000046.json | 12 +++ .../resources/schema/DUO/ValidDuoFile.json | 4 + .../src/test/resources/schema/DUO/duo.json | 77 +++++++++++++ 27 files changed, 589 insertions(+), 6 deletions(-) create mode 100644 services/workers/src/test/resources/schema/DUO/D0000001.json create mode 100644 services/workers/src/test/resources/schema/DUO/D0000004.json create mode 100644 services/workers/src/test/resources/schema/DUO/D0000006.json create mode 100644 services/workers/src/test/resources/schema/DUO/D0000007.json create mode 100644 services/workers/src/test/resources/schema/DUO/D0000011.json create mode 100644 services/workers/src/test/resources/schema/DUO/D0000012.json create mode 100644 services/workers/src/test/resources/schema/DUO/D0000015.json create mode 100644 services/workers/src/test/resources/schema/DUO/D0000016.json create mode 100644 services/workers/src/test/resources/schema/DUO/D0000018.json create mode 100644 services/workers/src/test/resources/schema/DUO/D0000019.json create mode 100644 services/workers/src/test/resources/schema/DUO/D0000020.json create mode 100644 services/workers/src/test/resources/schema/DUO/D0000021.json create mode 100644 services/workers/src/test/resources/schema/DUO/D0000022.json create mode 100644 services/workers/src/test/resources/schema/DUO/D0000024.json create mode 100644 services/workers/src/test/resources/schema/DUO/D0000025.json create mode 100644 services/workers/src/test/resources/schema/DUO/D0000026.json create mode 100644 services/workers/src/test/resources/schema/DUO/D0000027.json create mode 100644 services/workers/src/test/resources/schema/DUO/D0000028.json create mode 100644 services/workers/src/test/resources/schema/DUO/D0000029.json create mode 100644 services/workers/src/test/resources/schema/DUO/D0000042.json create mode 100644 services/workers/src/test/resources/schema/DUO/D0000043.json create mode 100644 services/workers/src/test/resources/schema/DUO/D0000044.json create mode 100644 services/workers/src/test/resources/schema/DUO/D0000045.json create mode 100644 services/workers/src/test/resources/schema/DUO/D0000046.json create mode 100644 services/workers/src/test/resources/schema/DUO/ValidDuoFile.json create mode 100644 services/workers/src/test/resources/schema/DUO/duo.json diff --git a/services/workers/src/test/java/org/sagebionetworks/schema/worker/JsonSchemaWorkerIntegrationTest.java b/services/workers/src/test/java/org/sagebionetworks/schema/worker/JsonSchemaWorkerIntegrationTest.java index e923b56230f..8dff35722c5 100644 --- a/services/workers/src/test/java/org/sagebionetworks/schema/worker/JsonSchemaWorkerIntegrationTest.java +++ b/services/workers/src/test/java/org/sagebionetworks/schema/worker/JsonSchemaWorkerIntegrationTest.java @@ -209,7 +209,7 @@ public String loadStringFromClasspath(String name) throws Exception { @Test public void testMainUseCase() throws Exception { - bootstrapAndCreateOrganization(); + bootstrapAndCreateOrganization("my.organization"); String[] schemasToRegister = { "pets/PetType.json", "pets/Pet.json", "pets/CatBreed.json", "pets/DogBreed.json", "pets/Cat.json", "pets/Dog.json", "pets/PetPhoto.json" }; for (String fileName : schemasToRegister) { @@ -252,11 +252,11 @@ public void testMainUseCase() throws Exception { printJson(result); } - void bootstrapAndCreateOrganization() throws RecoverableMessageException, InterruptedException { + void bootstrapAndCreateOrganization(String organizationName) throws RecoverableMessageException, InterruptedException { jsonSchemaManager.truncateAll(); schemaBootstrap.bootstrapSynapseSchemas(); CreateOrganizationRequest createOrgRequest = new CreateOrganizationRequest(); - createOrgRequest.setOrganizationName("my.organization"); + createOrgRequest.setOrganizationName(organizationName); organization = jsonSchemaManager.createOrganziation(adminUserInfo, createOrgRequest); } @@ -288,7 +288,7 @@ public void testGetValidationSchemaWorker() throws AssertionError, AsynchJobFail @Test public void testEntitySchemaValidation() throws Exception { - bootstrapAndCreateOrganization(); + bootstrapAndCreateOrganization("my.organization"); String projectId = entityManager.createEntity(adminUserInfo, new Project(), null); Project project = entityManager.getEntity(adminUserInfo, projectId, Project.class); @@ -352,7 +352,7 @@ public void testCreateWithDryRun() throws Exception { @Test public void testEntitySchemaValidationWithBoolean() throws Exception { - bootstrapAndCreateOrganization(); + bootstrapAndCreateOrganization("my.organization"); String projectId = entityManager.createEntity(adminUserInfo, new Project(), null); Project project = entityManager.getEntity(adminUserInfo, projectId, Project.class); @@ -397,7 +397,7 @@ public void testEntitySchemaValidationWithBoolean() throws Exception { @Test public void testNoSemanticVersionSchemaRevalidationWithSchemaChange() throws Exception { // PLFM-6757 - bootstrapAndCreateOrganization(); + bootstrapAndCreateOrganization("my.organization"); String projectId = entityManager.createEntity(adminUserInfo, new Project(), null); Project project = entityManager.getEntity(adminUserInfo, projectId, Project.class); @@ -683,6 +683,95 @@ public void testValidationSchemaIndexWithReindexingAndRevalidation() throws Exce waitForValidationResultsToBeNotFound(adminUserInfo, folderId); } + @Test + public void testDuoSchema() throws Exception { + bootstrapAndCreateOrganization("ebispot.duo"); + String[] schemasToRegister = { "schema/DUO/hmb.json", "schema/DUO/irb.json", "schema/DUO/duo.json"}; + for (String fileName : schemasToRegister) { + JsonSchema schema = getSchemaFromClasspath(fileName); + registerSchema(schema); + } + + JsonSchema validationSchema = jsonSchemaManager.getValidationSchema("ebispot.duo-duo"); + assertNotNull(schemaBootstrap); + printJson(validationSchema); + assertNotNull(validationSchema.getDefinitions()); + assertTrue(validationSchema.getDefinitions().containsKey("ebispot.duo-D0000006")); + assertTrue(validationSchema.getDefinitions().containsKey("ebispot.duo-D0000021")); + + + String validJsonFile = loadStringFromClasspath("schema/DUO/ValidDuoFile.json"); + JSONObject validJson = new JSONObject(validJsonFile); + JsonSubject mockSubject = Mockito.mock(JsonSubject.class); + when(mockSubject.toJson()).thenReturn(validJson); + // this schema should be valid + ValidationResults result = jsonSchemaValidationManager.validate(validationSchema, mockSubject); + assertNotNull(result); + assertTrue(result.getIsValid()); + } + + @Test + public void testDuoSchemaAppliedToFolder() throws Exception { + bootstrapAndCreateOrganization("ebispot.duo"); + String[] schemasToRegister = { + "schema/DUO/D0000001.json", + "schema/DUO/D0000004.json", + "schema/DUO/D0000006.json", + "schema/DUO/D0000007.json", + "schema/DUO/D0000011.json", + "schema/DUO/D0000012.json", + "schema/DUO/D0000015.json", + "schema/DUO/D0000016.json", + "schema/DUO/D0000018.json", + "schema/DUO/D0000019.json", + "schema/DUO/D0000020.json", + "schema/DUO/D0000021.json", + "schema/DUO/D0000022.json", + "schema/DUO/D0000024.json", + "schema/DUO/D0000025.json", + "schema/DUO/D0000026.json", + "schema/DUO/D0000027.json", + "schema/DUO/D0000028.json", + "schema/DUO/D0000029.json", + "schema/DUO/D0000042.json", + "schema/DUO/D0000043.json", + "schema/DUO/D0000044.json", + "schema/DUO/D0000045.json", + "schema/DUO/D0000046.json", + "schema/DUO/duo.json", + }; + for (String fileName : schemasToRegister) { + JsonSchema schema = getSchemaFromClasspath(fileName); + registerSchema(schema); + } + + JsonSchema validationSchema = jsonSchemaManager.getValidationSchema("ebispot.duo-duo"); + assertNotNull(schemaBootstrap); + printJson(validationSchema); + + String projectId = entityManager.createEntity(adminUserInfo, new Project(), null); + Project project = entityManager.getEntity(adminUserInfo, projectId, Project.class); + + String parentSchema$id = validationSchema.get$id(); + BindSchemaToEntityRequest bindRequest = new BindSchemaToEntityRequest(); + bindRequest.setEntityId(projectId); + bindRequest.setSchema$id(parentSchema$id); + // we want to validate on putting annotations, so don't send notifications + boolean sendNotificationMessages = false; + entityManager.bindSchemaToEntity(adminUserInfo, bindRequest, sendNotificationMessages); + + // add a folder to the project + Folder folder = new Folder(); + folder.setParentId(project.getId()); + String folderId = entityManager.createEntity(adminUserInfo, folder, null); + JSONObject folderJson = entityManager.getEntityJson(folderId); + + + folderJson.put("GRU", true); + folderJson = entityManager.updateEntityJson(adminUserInfo, folderId, folderJson); + System.out.println(folderJson.toString(5)); + } + /** * Wait for the validation results * diff --git a/services/workers/src/test/resources/schema/DUO/D0000001.json b/services/workers/src/test/resources/schema/DUO/D0000001.json new file mode 100644 index 00000000000..b16e8c180d7 --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000001.json @@ -0,0 +1,6 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000001", + "title": "data use permission", + "description":"A data item that is used to indicate consent permissions for datasets and/or materials, and relates to the purposes for which datasets and/or material might be removed, stored or used." +} diff --git a/services/workers/src/test/resources/schema/DUO/D0000004.json b/services/workers/src/test/resources/schema/DUO/D0000004.json new file mode 100644 index 00000000000..45c5b5841dd --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000004.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000004", + "title": "no restriction", + "properties": { + "NRES": { + "type": "boolean", + "description": "This data use permission indicates there is no restriction on use.", + "default": "false" + } + } +} diff --git a/services/workers/src/test/resources/schema/DUO/D0000006.json b/services/workers/src/test/resources/schema/DUO/D0000006.json new file mode 100644 index 00000000000..4f9aee46f1d --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000006.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000006", + "title": "health or medical or biomedical research", + "properties": { + "HMB": { + "type": "boolean", + "description": "This data use permission indicates that use is allowed for health/medical/biomedical purposes; does not include the study of population origins or ancestry.", + "default": "false" + } + } +} diff --git a/services/workers/src/test/resources/schema/DUO/D0000007.json b/services/workers/src/test/resources/schema/DUO/D0000007.json new file mode 100644 index 00000000000..263a8cbcc8a --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000007.json @@ -0,0 +1,33 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000007", + "title": "disease specific research", + "properties": { + "DS": { + "type": "boolean", + "description": "This data use permission indicates that use is allowed provided it is related to the specified disease.", + "default": "false" + } + }, + "if": { + "properties": { + "DS": { + "const": true + } + } + }, + "then": { + "properties": { + "DS_disease": { + "type": "string", + "description": "DUO recommends MONDO be used, to provide the basis for automated evaluation. For more information see https://github.com/EBISPOT/DUO/blob/master/MONDO_Overview.md", + "enum": [ + "cancer", + "alzheimer", + "amnesia", + "..." + ] + } + } + } +} diff --git a/services/workers/src/test/resources/schema/DUO/D0000011.json b/services/workers/src/test/resources/schema/DUO/D0000011.json new file mode 100644 index 00000000000..f0f522badc0 --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000011.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000011", + "title": "population origins or ancestry research only", + "properties": { + "POA": { + "type": "boolean", + "description": "This data use permission indicates that use of the data is limited to the study of population origins or ancestry.", + "default": "false" + } + } +} diff --git a/services/workers/src/test/resources/schema/DUO/D0000012.json b/services/workers/src/test/resources/schema/DUO/D0000012.json new file mode 100644 index 00000000000..0a0198d6ddc --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000012.json @@ -0,0 +1,31 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000012", + "title": "research specific restrictions", + "properties": { + "RS": { + "type": "boolean", + "description": "This data use modifier indicates that use is limited to studies of a certain research type.", + "default": "false" + } + }, + "if": { + "properties": { + "RS": { + "const": true + } + } + }, + "then": { + "properties": { + "RS_research_type": { + "type": "string", + "description": "...", + "enum": [ + "???", + "..." + ] + } + } + } +} diff --git a/services/workers/src/test/resources/schema/DUO/D0000015.json b/services/workers/src/test/resources/schema/DUO/D0000015.json new file mode 100644 index 00000000000..928b222f27f --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000015.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000015", + "title": "no general methods research", + "properties": { + "NMDS": { + "type": "boolean", + "description": "This data use modifier indicates that use does not allow methods development research (e.g., development of software or algorithms).", + "default": "false" + } + } +} diff --git a/services/workers/src/test/resources/schema/DUO/D0000016.json b/services/workers/src/test/resources/schema/DUO/D0000016.json new file mode 100644 index 00000000000..ddb15ac8f4f --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000016.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000016", + "title": "genetic studies only", + "properties": { + "GSO": { + "type": "boolean", + "description": "This data use modifier indicates that use is limited to genetic studies only (i.e., studies that include genotype research alone or both genotype and phenotype research, but not phenotype research exclusively)", + "default": "false" + } + } +} diff --git a/services/workers/src/test/resources/schema/DUO/D0000018.json b/services/workers/src/test/resources/schema/DUO/D0000018.json new file mode 100644 index 00000000000..547b591e6cf --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000018.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000018", + "title": "not for profit, non commercial use only", + "properties": { + "NPUNCU": { + "type": "boolean", + "description": "This data use modifier indicates that use of the data is limited to not-for-profit organizations and not-for-profit use, non-commercial use.", + "default": "false" + } + } +} diff --git a/services/workers/src/test/resources/schema/DUO/D0000019.json b/services/workers/src/test/resources/schema/DUO/D0000019.json new file mode 100644 index 00000000000..efb90647dbf --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000019.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000019", + "title": "publication required", + "properties": { + "PUB": { + "type": "boolean", + "description": "This data use modifier indicates that requestor agrees to make results of studies using the data available to the larger scientific community.", + "default": "false" + } + } +} diff --git a/services/workers/src/test/resources/schema/DUO/D0000020.json b/services/workers/src/test/resources/schema/DUO/D0000020.json new file mode 100644 index 00000000000..1cc6c954b79 --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000020.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000020", + "title": "collaboration required", + "properties": { + "COL": { + "type": "boolean", + "description": "This data use modifier indicates that the requestor must agree to collaboration with the primary study investigator(s).", + "default": "false" + } + }, + "if": { + "properties": { + "COL": { + "const": true + } + } + }, + "then": { + "properties": { + "COL_PI": { + "type": "string", + "description": "This could be coupled with a string describing the primary study investigator(s)." + } + } + } +} diff --git a/services/workers/src/test/resources/schema/DUO/D0000021.json b/services/workers/src/test/resources/schema/DUO/D0000021.json new file mode 100644 index 00000000000..db875a7240f --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000021.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000021", + "title": "ethics approval required", + "properties": { + "IRB": { + "type": "boolean", + "description": "This data use modifier indicates that the requestor must provide documentation of local IRB/ERB approval.", + "default": "false" + } + } +} diff --git a/services/workers/src/test/resources/schema/DUO/D0000022.json b/services/workers/src/test/resources/schema/DUO/D0000022.json new file mode 100644 index 00000000000..238181ce563 --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000022.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000022", + "title": "geographical restriction", + "properties": { + "GS": { + "type": "boolean", + "description": "This data use modifier indicates that use is limited to within a specific geographic region.", + "default": "false" + } + }, + "if": { + "properties": { + "GS": { + "const": true + } + } + }, + "then": { + "properties": { + "GS_location": { + "type": "string", + "description": "This should be coupled with an ontology term describing the geographical location the restriction applies to." + } + } + } +} diff --git a/services/workers/src/test/resources/schema/DUO/D0000024.json b/services/workers/src/test/resources/schema/DUO/D0000024.json new file mode 100644 index 00000000000..0029bf986ea --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000024.json @@ -0,0 +1,28 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000024", + "title": "publication moratorium", + "properties": { + "MOR": { + "type": "boolean", + "description": "This data use modifier indicates that requestor agrees not to publish results of studies until a specific date.", + "default": "false" + } + }, + "if": { + "properties": { + "MOR": { + "const": true + } + } + }, + "then": { + "properties": { + "MOR_date": { + "type": "string", + "format": "date-time", + "description": "This should be coupled with a date specified as ISO8601" + } + } + } +} diff --git a/services/workers/src/test/resources/schema/DUO/D0000025.json b/services/workers/src/test/resources/schema/DUO/D0000025.json new file mode 100644 index 00000000000..85a9410334b --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000025.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000025", + "title": "time limit on use", + "properties": { + "TS": { + "type": "boolean", + "description": "This data use modifier indicates that use is approved for a specific number of months.", + "default": "false" + } + }, + "if": { + "properties": { + "TS": { + "const": true + } + } + }, + "then": { + "properties": { + "TS_number_of_months": { + "type": "integer", + "description": "This should be coupled with an integer value indicating the number of months." + } + } + } +} diff --git a/services/workers/src/test/resources/schema/DUO/D0000026.json b/services/workers/src/test/resources/schema/DUO/D0000026.json new file mode 100644 index 00000000000..f7a9271f402 --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000026.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000026", + "title": "user specific restriction", + "properties": { + "US": { + "type": "boolean", + "description": "This data use modifier indicates that use is limited to use by approved users.", + "default": "false" + } + } +} diff --git a/services/workers/src/test/resources/schema/DUO/D0000027.json b/services/workers/src/test/resources/schema/DUO/D0000027.json new file mode 100644 index 00000000000..1601f7f95c5 --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000027.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000027", + "title": "project specific restriction", + "properties": { + "PS": { + "type": "boolean", + "description": "This data use modifier indicates that use is limited to use within an approved project.", + "default": "false" + } + }, + "if": { + "properties": { + "PS": { + "const": true + } + } + }, + "then": { + "properties": { + "PS_project": { + "type": "string", + "description": "???" + } + } + } +} diff --git a/services/workers/src/test/resources/schema/DUO/D0000028.json b/services/workers/src/test/resources/schema/DUO/D0000028.json new file mode 100644 index 00000000000..161d82c0770 --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000028.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000028", + "title": "institution specific restriction", + "properties": { + "IS": { + "type": "boolean", + "description": "This data use modifier indicates that use is limited to use within an approved institution.", + "default": "false" + } + }, + "if": { + "properties": { + "IS": { + "const": true + } + } + }, + "then": { + "properties": { + "IS_institution": { + "type": "string", + "description": "???" + } + } + } +} diff --git a/services/workers/src/test/resources/schema/DUO/D0000029.json b/services/workers/src/test/resources/schema/DUO/D0000029.json new file mode 100644 index 00000000000..fe6b58a13fb --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000029.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000029", + "title": "return to database or resource", + "properties": { + "RTN": { + "type": "boolean", + "description": "This data use modifier indicates that the requestor must return derived/enriched data to the database/resource.", + "default": "false" + } + } +} diff --git a/services/workers/src/test/resources/schema/DUO/D0000042.json b/services/workers/src/test/resources/schema/DUO/D0000042.json new file mode 100644 index 00000000000..dd6c216389f --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000042.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000042", + "title": "general research use", + "properties": { + "GRU": { + "type": "boolean", + "description": "This data use permission indicates that use is allowed for general research use for any research purpose.", + "default": "false" + } + } +} diff --git a/services/workers/src/test/resources/schema/DUO/D0000043.json b/services/workers/src/test/resources/schema/DUO/D0000043.json new file mode 100644 index 00000000000..dd40ce75ef6 --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000043.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000043", + "title": "clinical care use", + "properties": { + "CC": { + "type": "boolean", + "description": "This data use modifier indicates that use is allowed for clinical use and care. Clinical Care is defined as Health care or services provided at home, in a healthcare facility or hospital. Data may be used for clinical decision making.", + "default": "false" + } + } +} diff --git a/services/workers/src/test/resources/schema/DUO/D0000044.json b/services/workers/src/test/resources/schema/DUO/D0000044.json new file mode 100644 index 00000000000..7fc6c7046a6 --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000044.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000044", + "title": "population origins or ancestry research prohibited", + "properties": { + "NPOA": { + "type": "boolean", + "description": "This data use modifier indicates use for purposes of population, origin, or ancestry research is prohibited.", + "default": "false" + } + } +} diff --git a/services/workers/src/test/resources/schema/DUO/D0000045.json b/services/workers/src/test/resources/schema/DUO/D0000045.json new file mode 100644 index 00000000000..398b958a7d5 --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000045.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000045", + "title": "not for profit organisation use only", + "properties": { + "NPU": { + "type": "boolean", + "description": "This data use modifier indicates that use of the data is limited to not-for-profit organizations.", + "default": "false" + } + } +} diff --git a/services/workers/src/test/resources/schema/DUO/D0000046.json b/services/workers/src/test/resources/schema/DUO/D0000046.json new file mode 100644 index 00000000000..8ac5e2d7548 --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/D0000046.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "ebispot.duo-D0000046", + "title": "non-commercial use only", + "properties": { + "NCU": { + "type": "boolean", + "description": "This data use modifier indicates that use of the data is limited to not-for-profit use. This indicates that data can be used by commercial organisations for research purposes, but not commercial purposes.", + "default": "false" + } + } +} diff --git a/services/workers/src/test/resources/schema/DUO/ValidDuoFile.json b/services/workers/src/test/resources/schema/DUO/ValidDuoFile.json new file mode 100644 index 00000000000..82280af5844 --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/ValidDuoFile.json @@ -0,0 +1,4 @@ +{ + "health or medical or biomedical research": true, + "ethics approval required": true +} \ No newline at end of file diff --git a/services/workers/src/test/resources/schema/DUO/duo.json b/services/workers/src/test/resources/schema/DUO/duo.json new file mode 100644 index 00000000000..e4fbf785b74 --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/duo.json @@ -0,0 +1,77 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "title": "Full DUO schema", + "$id": "ebispot.duo-duo", + "description": "...", + "allOf": [ + { + "$ref": "ebispot.duo-D0000004" + }, + { + "$ref": "ebispot.duo-D0000006" + }, + { + "$ref": "ebispot.duo-D0000007" + }, + { + "$ref": "ebispot.duo-D0000011" + }, + { + "$ref": "ebispot.duo-D0000012" + }, + { + "$ref": "ebispot.duo-D0000015" + }, + { + "$ref": "ebispot.duo-D0000016" + }, + { + "$ref": "ebispot.duo-D0000018" + }, + { + "$ref": "ebispot.duo-D0000019" + }, + { + "$ref": "ebispot.duo-D0000020" + }, + { + "$ref": "ebispot.duo-D0000021" + }, + { + "$ref": "ebispot.duo-D0000022" + }, + { + "$ref": "ebispot.duo-D0000024" + }, + { + "$ref": "ebispot.duo-D0000025" + }, + { + "$ref": "ebispot.duo-D0000026" + }, + { + "$ref": "ebispot.duo-D0000027" + }, + { + "$ref": "ebispot.duo-D0000028" + }, + { + "$ref": "ebispot.duo-D0000029" + }, + { + "$ref": "ebispot.duo-D0000042" + }, + { + "$ref": "ebispot.duo-D0000043" + }, + { + "$ref": "ebispot.duo-D0000044" + }, + { + "$ref": "ebispot.duo-D0000045" + }, + { + "$ref": "ebispot.duo-D0000046" + } + ] +} From 285c9442c57d6bace824db4e8ebaea84a91fcb9b Mon Sep 17 00:00:00 2001 From: john-hill Date: Tue, 12 Apr 2022 13:01:01 -0700 Subject: [PATCH 2/3] added an example project schema using DUO and examples of valid JSON --- .../JsonSchemaWorkerIntegrationTest.java | 46 +++--- .../test/resources/schema/DUO/D0000012.json | 2 +- .../test/resources/schema/DUO/D0000024.json | 2 +- .../resources/schema/DUO/DuoMainStory.json | 146 ++++++++++++++++++ .../test/resources/schema/DUO/ValidSyn1.json | 45 ++++++ .../test/resources/schema/DUO/ValidSyn4.json | 47 ++++++ 6 files changed, 261 insertions(+), 27 deletions(-) create mode 100644 services/workers/src/test/resources/schema/DUO/DuoMainStory.json create mode 100644 services/workers/src/test/resources/schema/DUO/ValidSyn1.json create mode 100644 services/workers/src/test/resources/schema/DUO/ValidSyn4.json diff --git a/services/workers/src/test/java/org/sagebionetworks/schema/worker/JsonSchemaWorkerIntegrationTest.java b/services/workers/src/test/java/org/sagebionetworks/schema/worker/JsonSchemaWorkerIntegrationTest.java index 8dff35722c5..b4dcb63266f 100644 --- a/services/workers/src/test/java/org/sagebionetworks/schema/worker/JsonSchemaWorkerIntegrationTest.java +++ b/services/workers/src/test/java/org/sagebionetworks/schema/worker/JsonSchemaWorkerIntegrationTest.java @@ -711,8 +711,9 @@ public void testDuoSchema() throws Exception { } @Test - public void testDuoSchemaAppliedToFolder() throws Exception { + public void testDuoSchemaAppliedAgainstValidFileJson() throws Exception { bootstrapAndCreateOrganization("ebispot.duo"); + jsonSchemaManager.createOrganziation(adminUserInfo, new CreateOrganizationRequest().setOrganizationName("some.project")); String[] schemasToRegister = { "schema/DUO/D0000001.json", "schema/DUO/D0000004.json", @@ -739,37 +740,32 @@ public void testDuoSchemaAppliedToFolder() throws Exception { "schema/DUO/D0000045.json", "schema/DUO/D0000046.json", "schema/DUO/duo.json", + "schema/DUO/DuoMainStory.json", }; for (String fileName : schemasToRegister) { JsonSchema schema = getSchemaFromClasspath(fileName); registerSchema(schema); } - - JsonSchema validationSchema = jsonSchemaManager.getValidationSchema("ebispot.duo-duo"); - assertNotNull(schemaBootstrap); - printJson(validationSchema); - - String projectId = entityManager.createEntity(adminUserInfo, new Project(), null); - Project project = entityManager.getEntity(adminUserInfo, projectId, Project.class); - - String parentSchema$id = validationSchema.get$id(); - BindSchemaToEntityRequest bindRequest = new BindSchemaToEntityRequest(); - bindRequest.setEntityId(projectId); - bindRequest.setSchema$id(parentSchema$id); - // we want to validate on putting annotations, so don't send notifications - boolean sendNotificationMessages = false; - entityManager.bindSchemaToEntity(adminUserInfo, bindRequest, sendNotificationMessages); - - // add a folder to the project - Folder folder = new Folder(); - folder.setParentId(project.getId()); - String folderId = entityManager.createEntity(adminUserInfo, folder, null); - JSONObject folderJson = entityManager.getEntityJson(folderId); + JsonSchema duoMain = jsonSchemaManager.getValidationSchema("ebispot.duo-duo"); + assertNotNull(duoMain); + printJson(duoMain); - folderJson.put("GRU", true); - folderJson = entityManager.updateEntityJson(adminUserInfo, folderId, folderJson); - System.out.println(folderJson.toString(5)); + JsonSchema validationSchema = jsonSchemaManager.getValidationSchema("some.project-main"); + assertNotNull(validationSchema); + printJson(validationSchema); + + String[] validJsonFiles = { "schema/DUO/ValidSyn1.json", "schema/DUO/ValidSyn4.json" }; + for (String schemaFile : validJsonFiles) { + String validJsonFile = loadStringFromClasspath(schemaFile); + JSONObject validJson = new JSONObject(validJsonFile); + JsonSubject mockSubject = Mockito.mock(JsonSubject.class); + when(mockSubject.toJson()).thenReturn(validJson); + // this schema should be valid + ValidationResults result = jsonSchemaValidationManager.validate(validationSchema, mockSubject); + assertNotNull(result); + assertTrue(result.getIsValid()); + } } /** diff --git a/services/workers/src/test/resources/schema/DUO/D0000012.json b/services/workers/src/test/resources/schema/DUO/D0000012.json index 0a0198d6ddc..a2b3c329980 100644 --- a/services/workers/src/test/resources/schema/DUO/D0000012.json +++ b/services/workers/src/test/resources/schema/DUO/D0000012.json @@ -22,7 +22,7 @@ "type": "string", "description": "...", "enum": [ - "???", + "cancer", "..." ] } diff --git a/services/workers/src/test/resources/schema/DUO/D0000024.json b/services/workers/src/test/resources/schema/DUO/D0000024.json index 0029bf986ea..847ee879952 100644 --- a/services/workers/src/test/resources/schema/DUO/D0000024.json +++ b/services/workers/src/test/resources/schema/DUO/D0000024.json @@ -20,7 +20,7 @@ "properties": { "MOR_date": { "type": "string", - "format": "date-time", + "format": "date", "description": "This should be coupled with a date specified as ISO8601" } } diff --git a/services/workers/src/test/resources/schema/DUO/DuoMainStory.json b/services/workers/src/test/resources/schema/DUO/DuoMainStory.json new file mode 100644 index 00000000000..aed18224bd5 --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/DuoMainStory.json @@ -0,0 +1,146 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "title": "Schema for Some Project", + "$id": "some.project-main", + "description": "This schema defines how DUO should be used with Some Project.", + "allOf": [ + { + "$ref": "org.sagebionetworks-repo.model.FileEntity-1.0.0" + }, + { + "$ref": "ebispot.duo-duo" + }, + { + "if": { + "properties": { + "patientLocation": { + "const": "Germany" + }, + "assayType": { + "const": "genomic" + } + } + }, + "then": { + "properties": { + "GS": { + "title": "geographical restriction", + "type": "boolean", + "const": true + }, + "GS_location": { + "type": "string", + "description": "This data cannot leave Germany", + "const": "Germany" + } + }, + "required": [ + "GS_location" + ] + } + }, + { + "if": { + "properties": { + "patientLocation": { + "const": "USA" + }, + "assayType": { + "const": "genomic" + } + } + }, + "then": { + "properties": { + "sourceGeography": { + "const": "US" + }, + "jurisdiction ": { + "const": "HIPAA" + }, + "dataLabel": { + "const": "De-identified" + } + }, + "required": [ + "sourceGeography", + "jurisdiction", + "dataLabel" + ] + } + } + ], + "properties": { + "assayType": { + "description": "Identifies they type of data for this files.", + "type": "string", + "enum": [ + "clinical", + "assay", + "imaging", + "genomic" + ] + }, + "patientLocation": { + "description": "The location of the patient associated with the data", + "type": "string", + "enum": [ + "USA", + "Germany" + ] + }, + "RS": { + "title": "research specific restrictions", + "type": "boolean", + "const": true + }, + "RS_research_type": { + "title": "Restricted to cancer research", + "type": "string", + "const": "cancer" + }, + "IRB": { + "title": "ethics approval required", + "type": "boolean", + "const": true + }, + "MOR": { + "title": "publication moratorium", + "type": "boolean", + "const": true + }, + "MOR_date": { + "title": "publication moratorium date", + "type": "string", + "format": "date", + "const": "2022-05-20" + } + }, + "required": [ + "assayType", + "patientLocation", + "NRES", + "HMB", + "DS", + "POA", + "RS", + "NMDS", + "GSO", + "NPUNCU", + "PUB", + "COL", + "IRB", + "GS", + "MOR", + "TS", + "US", + "PS", + "IS", + "RTN", + "GRU", + "CC", + "NPOA", + "NPU", + "NCU" + ] +} diff --git a/services/workers/src/test/resources/schema/DUO/ValidSyn1.json b/services/workers/src/test/resources/schema/DUO/ValidSyn1.json new file mode 100644 index 00000000000..b95419ce817 --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/ValidSyn1.json @@ -0,0 +1,45 @@ +{ + "name": "GermanGenomic.data", + "description": "Genomic data from patients in Germany", + "id": "syn1", + "etag": "some-etag", + "createdOn": "2020-05-20T20:20:39+00:00", + "modifiedOn": "2020-05-20T20:20:39+00:00", + "createdBy": "123456789", + "modifiedBy": "123456789", + "parentId": "syn444", + "versionLabel": "one", + "versionComment": "leaving blank", + "versionNumber": 1, + "dataFileHandleId": "98765", + "fileNameOverride": "", + "concreteType": "org.sagebionetworks.repo.model.FileEntity", + "assayType": "genomic", + "patientLocation": "Germany", + "NRES": false, + "HMB": false, + "DS": false, + "POA": false, + "RS": true, + "RS_research_type": "cancer", + "NMDS": false, + "GSO": false, + "NPUNCU": false, + "PUB": false, + "COL": false, + "IRB": true, + "GS": true, + "GS_location": "Germany", + "MOR": true, + "MOR_date": "2022-05-20", + "TS": false, + "US": false, + "PS": false, + "IS": false, + "RTN": false, + "GRU": false, + "CC": false, + "NPOA": false, + "NPU": false, + "NCU": false +} \ No newline at end of file diff --git a/services/workers/src/test/resources/schema/DUO/ValidSyn4.json b/services/workers/src/test/resources/schema/DUO/ValidSyn4.json new file mode 100644 index 00000000000..50d2afb8105 --- /dev/null +++ b/services/workers/src/test/resources/schema/DUO/ValidSyn4.json @@ -0,0 +1,47 @@ +{ + "name": "USGenomic.data", + "description": "Genomic data from patients in USA", + "id": "syn4", + "etag": "some-etag", + "createdOn": "2020-05-20T20:20:39+00:00", + "modifiedOn": "2020-05-20T20:20:39+00:00", + "createdBy": "123456789", + "modifiedBy": "123456789", + "parentId": "syn444", + "versionLabel": "one", + "versionComment": "leaving blank", + "versionNumber": 1, + "dataFileHandleId": "98765", + "fileNameOverride": "", + "concreteType": "org.sagebionetworks.repo.model.FileEntity", + "assayType": "genomic", + "patientLocation": "USA", + "NRES": false, + "HMB": false, + "DS": false, + "POA": false, + "RS": true, + "RS_research_type": "cancer", + "NMDS": false, + "GSO": false, + "NPUNCU": false, + "PUB": false, + "COL": false, + "IRB": true, + "GS": false, + "MOR": true, + "MOR_date": "2022-05-20", + "TS": false, + "US": false, + "PS": false, + "IS": false, + "RTN": false, + "GRU": false, + "CC": false, + "NPOA": false, + "NPU": false, + "NCU": false, + "sourceGeography":"US", + "jurisdiction": "HIPAA", + "dataLabel":"De-identified" +} \ No newline at end of file From 428155033289a7a364c22b29ef4389fbfec84fb4 Mon Sep 17 00:00:00 2001 From: john-hill Date: Wed, 27 Apr 2022 15:35:55 -0700 Subject: [PATCH 3/3] done for now --- .../test/resources/schema/DUO/DuoMainStory.json | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/services/workers/src/test/resources/schema/DUO/DuoMainStory.json b/services/workers/src/test/resources/schema/DUO/DuoMainStory.json index aed18224bd5..d5cf19931af 100644 --- a/services/workers/src/test/resources/schema/DUO/DuoMainStory.json +++ b/services/workers/src/test/resources/schema/DUO/DuoMainStory.json @@ -60,6 +60,11 @@ }, "dataLabel": { "const": "De-identified" + }, + "_accessRequirementIds": { + "const": [ + 4 + ] } }, "required": [ @@ -114,6 +119,13 @@ "type": "string", "format": "date", "const": "2022-05-20" + }, + "_accessRequirementIds": { + "const": [ + 1, + 2, + 3 + ] } }, "required": [ @@ -143,4 +155,4 @@ "NPU", "NCU" ] -} +} \ No newline at end of file