-
Notifications
You must be signed in to change notification settings - Fork 97
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
TASK-1562 - Pedigree graph with a three-generation family fails to render properly in Family browser detailed tab #2253
Merged
Changes from 15 commits
Commits
Show all changes
22 commits
Select commit
Hold shift + click to select a range
b9429dd
analysis: add family analysis to compute the pedigree graph image, #T…
jtarraga e5d2400
client: generate CLI, #TASK-3898, #TASK-1562
jtarraga e0187ce
analysis: add R script and update opencga-ext-tools dockerfile, #TASK…
jtarraga f4c7dc2
analysis: update executor annotation, #TASK-3898, #TASK-1562
jtarraga 72cbfe3
analysis: remove temporary/scratch directory, #TASK-3898, #TASK-1562
jtarraga eb83ecf
analysis: set to 0 when missing life status, #TASK-3898, #TASK-1562
jtarraga 02c0211
analysis: update R script, #TASK-3898, #TASK-1562
jtarraga 2d42c04
catalog: save pedigree graph image in format Base64 in the Family dat…
jtarraga 52a5fb9
catalog: fix checkstyle issues, #TASK-4000, #TASK-1562
jtarraga 009a693
Merge branch 'develop' into TASK-1562
jtarraga d513a96
Merge branch 'develop' into TASK-1562
imedina d853fb0
Merge branch 'develop' into TASK-1562
imedina e7a22a9
catalog: compute family pedigree graph when updating disorders, and i…
jtarraga a3af77b
Merge branch 'TASK-1562' of https://github.com/opencb/opencga into TA…
jtarraga fa370cb
catalog: update R script and save the pedigree graph in JSON format i…
jtarraga da7848f
app: calculate pedigree graph for all the families, #TASK-4067
pfurio 0d687dc
migration: add pedigree graph init analysis to initialize all familie…
jtarraga eda5aac
app: R script completes pedigree by adding the missing parent, #TASK-…
jtarraga f7dfab9
catalog: add some checks before compute the pedigree graph, #TASK-400…
jtarraga 5562b58
Merge branch 'develop' into TASK-1562
jtarraga 35a6289
analysis: support 2-member 2-generation families, #TASK-3898, #TASK-1562
jtarraga 892c96c
Resolve conflicts after merging the branch 'develop', #TASK-1562
jtarraga File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
77 changes: 77 additions & 0 deletions
77
opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/PedigreeGraphAnalysis.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,77 @@ | ||
/* | ||
* Copyright 2015-2020 OpenCB | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package org.opencb.opencga.analysis.family; | ||
|
||
import org.apache.commons.lang3.StringUtils; | ||
import org.opencb.commons.datastore.core.QueryOptions; | ||
import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy; | ||
import org.opencb.opencga.core.exceptions.ToolException; | ||
import org.opencb.opencga.core.models.common.Enums; | ||
import org.opencb.opencga.core.models.family.Family; | ||
import org.opencb.opencga.core.models.family.PedigreeGraphAnalysisParams; | ||
import org.opencb.opencga.core.response.OpenCGAResult; | ||
import org.opencb.opencga.core.tools.annotations.Tool; | ||
import org.opencb.opencga.core.tools.annotations.ToolParams; | ||
import org.opencb.opencga.core.tools.family.PedigreeGraphAnalysisExecutor; | ||
|
||
@Tool(id = PedigreeGraphAnalysis.ID, resource = Enums.Resource.FAMILY) | ||
public class PedigreeGraphAnalysis extends OpenCgaToolScopeStudy { | ||
|
||
public static final String ID = "pedigree-graph"; | ||
public static final String DESCRIPTION = "Compute the family pedigree graph image."; | ||
|
||
@ToolParams | ||
private PedigreeGraphAnalysisParams pedigreeParams = new PedigreeGraphAnalysisParams(); | ||
|
||
private Family family; | ||
|
||
@Override | ||
protected void check() throws Exception { | ||
super.check(); | ||
setUpStorageEngineExecutor(study); | ||
|
||
if (StringUtils.isEmpty(getStudy())) { | ||
throw new ToolException("Missing study"); | ||
} | ||
|
||
if (StringUtils.isEmpty(pedigreeParams.getFamilyId())) { | ||
throw new ToolException("Missing family ID. It is mandatory to compute the pedigree graph image"); | ||
} | ||
|
||
// Check family | ||
study = catalogManager.getStudyManager().get(study, QueryOptions.empty(), token).first().getFqn(); | ||
OpenCGAResult<Family> familyResult = catalogManager.getFamilyManager().get(study, pedigreeParams.getFamilyId(), | ||
QueryOptions.empty(), token); | ||
if (familyResult.getNumResults() != 1) { | ||
throw new ToolException("Unable to compute the pedigree graph imae. Family '" + pedigreeParams.getFamilyId() + "' not found"); | ||
} | ||
family = familyResult.first(); | ||
|
||
} | ||
|
||
@Override | ||
protected void run() throws ToolException { | ||
step(getId(), () -> { | ||
PedigreeGraphAnalysisExecutor toolExecutor = getToolExecutor(PedigreeGraphAnalysisExecutor.class); | ||
|
||
toolExecutor.setStudy(study) | ||
.setFamily(family) | ||
.execute(); | ||
}); | ||
} | ||
} | ||
|
58 changes: 58 additions & 0 deletions
58
.../src/main/java/org/opencb/opencga/analysis/family/PedigreeGraphLocalAnalysisExecutor.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
/* | ||
* Copyright 2015-2020 OpenCB | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package org.opencb.opencga.analysis.family; | ||
|
||
import org.apache.commons.collections4.CollectionUtils; | ||
import org.apache.commons.lang3.StringUtils; | ||
import org.opencb.biodata.models.clinical.Disorder; | ||
import org.opencb.commons.utils.DockerUtils; | ||
import org.opencb.opencga.analysis.StorageToolExecutor; | ||
import org.opencb.opencga.catalog.exceptions.CatalogException; | ||
import org.opencb.opencga.catalog.utils.PedigreeGraphUtils; | ||
import org.opencb.opencga.core.common.GitRepositoryState; | ||
import org.opencb.opencga.core.exceptions.ToolException; | ||
import org.opencb.opencga.core.models.individual.Individual; | ||
import org.opencb.opencga.core.tools.annotations.ToolExecutor; | ||
import org.opencb.opencga.core.tools.family.PedigreeGraphAnalysisExecutor; | ||
import org.opencb.opencga.storage.core.exceptions.StorageEngineException; | ||
import org.slf4j.Logger; | ||
import org.slf4j.LoggerFactory; | ||
|
||
import java.io.File; | ||
import java.io.FileNotFoundException; | ||
import java.io.IOException; | ||
import java.io.PrintWriter; | ||
import java.nio.file.Path; | ||
import java.nio.file.Paths; | ||
import java.util.*; | ||
|
||
@ToolExecutor(id="opencga-local", tool = PedigreeGraphAnalysis.ID, | ||
framework = ToolExecutor.Framework.LOCAL, source = ToolExecutor.Source.STORAGE) | ||
public class PedigreeGraphLocalAnalysisExecutor extends PedigreeGraphAnalysisExecutor { | ||
|
||
private Path opencgaHome; | ||
|
||
private Logger logger = LoggerFactory.getLogger(this.getClass()); | ||
|
||
@Override | ||
public void run() throws ToolException, CatalogException, IOException, StorageEngineException { | ||
opencgaHome = Paths.get(getExecutorParams().getString("opencgaHome")); | ||
|
||
// Run R script for fitting signature | ||
PedigreeGraphUtils.createPedigreeGraph(getFamily(), opencgaHome.resolve("analysis/" + PedigreeGraphAnalysis.ID), getOutDir()); | ||
} | ||
} |
220 changes: 220 additions & 0 deletions
220
opencga-analysis/src/test/java/org/opencb/opencga/analysis/family/FamilyAnalysisTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,220 @@ | ||
package org.opencb.opencga.analysis.family; | ||
|
||
import org.junit.*; | ||
import org.junit.rules.ExpectedException; | ||
import org.opencb.biodata.models.clinical.Disorder; | ||
import org.opencb.biodata.models.clinical.Phenotype; | ||
import org.opencb.biodata.models.core.SexOntologyTermAnnotation; | ||
import org.opencb.biodata.models.pedigree.IndividualProperty; | ||
import org.opencb.commons.datastore.core.DataResult; | ||
import org.opencb.commons.datastore.core.ObjectMap; | ||
import org.opencb.commons.datastore.core.QueryOptions; | ||
import org.opencb.commons.test.GenericTest; | ||
import org.opencb.opencga.TestParamConstants; | ||
import org.opencb.opencga.analysis.tools.ToolRunner; | ||
import org.opencb.opencga.analysis.variant.OpenCGATestExternalResource; | ||
import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; | ||
import org.opencb.opencga.catalog.exceptions.CatalogException; | ||
import org.opencb.opencga.catalog.managers.CatalogManager; | ||
import org.opencb.opencga.catalog.managers.FamilyManager; | ||
import org.opencb.opencga.catalog.utils.PedigreeGraphUtils; | ||
import org.opencb.opencga.core.api.ParamConstants; | ||
import org.opencb.opencga.core.exceptions.ToolException; | ||
import org.opencb.opencga.core.models.family.*; | ||
import org.opencb.opencga.core.models.individual.Individual; | ||
import org.opencb.opencga.core.models.individual.IndividualUpdateParams; | ||
import org.opencb.opencga.core.models.sample.Sample; | ||
import org.opencb.opencga.core.models.sample.SampleReferenceParam; | ||
import org.opencb.opencga.core.models.user.Account; | ||
import org.opencb.opencga.core.response.OpenCGAResult; | ||
import org.opencb.opencga.storage.core.StorageEngineFactory; | ||
|
||
import java.io.IOException; | ||
import java.nio.file.Path; | ||
import java.nio.file.Paths; | ||
import java.util.*; | ||
|
||
import static org.junit.Assert.*; | ||
|
||
public class FamilyAnalysisTest extends GenericTest { | ||
|
||
public final static String STUDY = "user@1000G:phase1"; | ||
@Rule | ||
public ExpectedException thrown = ExpectedException.none(); | ||
|
||
@ClassRule | ||
public static OpenCGATestExternalResource opencga = new OpenCGATestExternalResource(); | ||
|
||
protected CatalogManager catalogManager; | ||
private FamilyManager familyManager; | ||
private String opencgaToken; | ||
protected String sessionIdUser; | ||
|
||
protected Family family; | ||
|
||
protected String projectId; | ||
protected String studyId; | ||
|
||
private static final QueryOptions INCLUDE_RESULT = new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true); | ||
|
||
@Before | ||
public void setUp() throws IOException, CatalogException { | ||
catalogManager = opencga.getCatalogManager(); | ||
familyManager = catalogManager.getFamilyManager(); | ||
setUpCatalogManager(catalogManager); | ||
} | ||
|
||
public void setUpCatalogManager(CatalogManager catalogManager) throws CatalogException { | ||
opencgaToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); | ||
|
||
catalogManager.getUserManager().create("user", "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, Account.AccountType.FULL, opencgaToken); | ||
sessionIdUser = catalogManager.getUserManager().login("user", TestParamConstants.PASSWORD).getToken(); | ||
|
||
projectId = catalogManager.getProjectManager().create("1000G", "Project about some genomes", "", "Homo sapiens", null, "GRCh38", INCLUDE_RESULT, sessionIdUser).first().getId(); | ||
studyId = catalogManager.getStudyManager().create(projectId, "phase1", null, "Phase 1", "Done", null, null, null, null, null, sessionIdUser).first().getId(); | ||
|
||
family = createDummyFamily("Martinez-Martinez", true).first(); | ||
} | ||
|
||
@After | ||
public void tearDown() throws Exception { | ||
} | ||
|
||
@Test | ||
public void creationTest() { | ||
PedigreeGraph pedigreeGraph = family.getPedigreeGraph(); | ||
assertTrue(pedigreeGraph.getBase64().startsWith("iVBORw0KGgoAAAANSUhEUgAAAeAAAAHgCAM")); | ||
assertTrue(pedigreeGraph.getBase64().endsWith("U949gmplQAAAABJRU5ErkJggg==")); | ||
} | ||
|
||
@Test | ||
public void updateTest() throws CatalogException { | ||
FamilyUpdateParams updateParams = new FamilyUpdateParams(); | ||
|
||
QueryOptions queryOptions = new QueryOptions() | ||
.append(ParamConstants.FAMILY_UPDATE_ROLES_PARAM, true) | ||
.append(ParamConstants.INCLUDE_RESULT_PARAM, true); | ||
Family updatedFamily = catalogManager.getFamilyManager().update(studyId, family.getId(), updateParams, queryOptions, sessionIdUser).first(); | ||
|
||
PedigreeGraph pedigreeGraph = updatedFamily.getPedigreeGraph(); | ||
assertTrue(pedigreeGraph.getBase64().startsWith("iVBORw0KGgoAAAANSUhEUgAAAeAAAAHgCAM")); | ||
assertTrue(pedigreeGraph.getBase64().endsWith("U949gmplQAAAABJRU5ErkJggg==")); | ||
} | ||
|
||
@Test | ||
public void testPedigreeGraphAnalysis() throws ToolException, IOException { | ||
Path outDir = Paths.get(opencga.createTmpOutdir("_pedigree_graph")); | ||
System.out.println("out dir = " + outDir.toAbsolutePath()); | ||
System.out.println("opencga home = " + opencga.getOpencgaHome().toAbsolutePath()); | ||
System.out.println(Paths.get("workspace parent = " + catalogManager.getConfiguration().getWorkspace()).getParent()); | ||
|
||
VariantStorageManager variantStorageManager = new VariantStorageManager(catalogManager, opencga.getStorageEngineFactory()); | ||
ToolRunner toolRunner = new ToolRunner(opencga.getOpencgaHome().toString(), catalogManager, | ||
StorageEngineFactory.get(variantStorageManager.getStorageConfiguration())); | ||
|
||
// Pedigree graph params | ||
PedigreeGraphAnalysisParams params = new PedigreeGraphAnalysisParams(); | ||
params.setFamilyId(family.getId()); | ||
|
||
toolRunner.execute(PedigreeGraphAnalysis.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, studyId), outDir, null, | ||
sessionIdUser); | ||
|
||
String b64Image = PedigreeGraphUtils.getB64Image(outDir); | ||
assertTrue(b64Image.startsWith("iVBORw0KGgoAAAANSUhEUgAAAeAAAAHgCAM")); | ||
assertTrue(b64Image.endsWith("U949gmplQAAAABJRU5ErkJggg==")); | ||
|
||
String tsvPedigreeGraph = PedigreeGraphUtils.getTsvPedigreeGraph(outDir); | ||
assertTrue(tsvPedigreeGraph.startsWith("id\tdadid\tmomid\tsex\taffected")); | ||
assertTrue(tsvPedigreeGraph.endsWith("2\t1\t2\t1\tNA\t2\t2\n")); | ||
|
||
assertEquals(family.getPedigreeGraph().getBase64(), b64Image); | ||
} | ||
|
||
private DataResult<Family> createDummyFamily(String familyName, boolean createMissingMembers) throws CatalogException { | ||
if (createMissingMembers) { | ||
Sample sample1 = new Sample().setId("sample1"); | ||
catalogManager.getSampleManager().create(STUDY, sample1, QueryOptions.empty(), sessionIdUser); | ||
|
||
Sample sample2 = new Sample().setId("sample2"); | ||
catalogManager.getSampleManager().create(STUDY, sample2, QueryOptions.empty(), sessionIdUser); | ||
|
||
Sample sample3 = new Sample().setId("sample3"); | ||
catalogManager.getSampleManager().create(STUDY, sample3, QueryOptions.empty(), sessionIdUser); | ||
} | ||
|
||
Phenotype phenotype1 = new Phenotype("dis1", "Phenotype 1", "HPO"); | ||
Phenotype phenotype2 = new Phenotype("dis2", "Phenotype 2", "HPO"); | ||
|
||
Disorder disorder1 = new Disorder("disorder-1", null, null, null, null, null, null); | ||
Disorder disorder2 = new Disorder("disorder-2", null, null, null, null, null, null); | ||
|
||
Individual father = new Individual().setId("father") | ||
.setPhenotypes(Arrays.asList(phenotype1)) | ||
.setSex(SexOntologyTermAnnotation.initMale()) | ||
.setLifeStatus(IndividualProperty.LifeStatus.ALIVE) | ||
.setDisorders(Collections.singletonList(disorder1)); | ||
|
||
Individual mother = new Individual().setId("mother") | ||
.setPhenotypes(Arrays.asList(phenotype2)) | ||
.setSex(SexOntologyTermAnnotation.initFemale()) | ||
.setLifeStatus(IndividualProperty.LifeStatus.ALIVE); | ||
|
||
// // We create a new father and mother with the same information to mimic the behaviour of the webservices. Otherwise, we would be | ||
// // ingesting references to exactly the same object and this test would not work exactly the same way. | ||
Individual relFather = new Individual().setId("father").setPhenotypes(Arrays.asList(phenotype1)); | ||
Individual relMother = new Individual().setId("mother").setPhenotypes(Arrays.asList(phenotype2)); | ||
|
||
Individual relChild1 = new Individual().setId("child1") | ||
.setPhenotypes(Arrays.asList(phenotype1, phenotype2)) | ||
.setFather(father) | ||
.setMother(mother) | ||
.setSex(SexOntologyTermAnnotation.initMale()) | ||
.setLifeStatus(IndividualProperty.LifeStatus.ALIVE) | ||
.setParentalConsanguinity(true); | ||
|
||
Individual relChild2 = new Individual().setId("child2") | ||
.setPhenotypes(Arrays.asList(phenotype1)) | ||
.setFather(father) | ||
.setMother(mother) | ||
.setSex(SexOntologyTermAnnotation.initFemale()) | ||
.setLifeStatus(IndividualProperty.LifeStatus.ALIVE) | ||
.setDisorders(Collections.singletonList(disorder1)) | ||
.setParentalConsanguinity(true); | ||
|
||
Individual relChild3 = new Individual().setId("child3") | ||
.setPhenotypes(Arrays.asList(phenotype1)) | ||
.setFather(father) | ||
.setMother(mother) | ||
.setSex(SexOntologyTermAnnotation.initFemale()) | ||
.setLifeStatus(IndividualProperty.LifeStatus.DECEASED) | ||
.setDisorders(Collections.singletonList(disorder2)) | ||
.setParentalConsanguinity(true); | ||
|
||
List<Individual> members = null; | ||
List<String> memberIds = null; | ||
if (createMissingMembers) { | ||
members = Arrays.asList(relChild1, relChild2, relChild3, father, mother); | ||
} else { | ||
memberIds = Arrays.asList("father", "mother", "child1", "child2", "child3"); | ||
} | ||
|
||
Family family = new Family(familyName, familyName, null, null, members, "", 5, | ||
Collections.emptyList(), Collections.emptyMap()); | ||
|
||
OpenCGAResult<Family> familyOpenCGAResult = familyManager.create(STUDY, family, memberIds, INCLUDE_RESULT, sessionIdUser); | ||
|
||
if (createMissingMembers) { | ||
catalogManager.getIndividualManager().update(STUDY, relChild1.getId(), | ||
new IndividualUpdateParams().setSamples(Collections.singletonList(new SampleReferenceParam().setId("sample1"))), | ||
QueryOptions.empty(), sessionIdUser); | ||
catalogManager.getIndividualManager().update(STUDY, relFather.getId(), | ||
new IndividualUpdateParams().setSamples(Collections.singletonList(new SampleReferenceParam().setId("sample2"))), | ||
QueryOptions.empty(), sessionIdUser); | ||
catalogManager.getIndividualManager().update(STUDY, relMother.getId(), | ||
new IndividualUpdateParams().setSamples(Collections.singletonList(new SampleReferenceParam().setId("sample3"))), | ||
QueryOptions.empty(), sessionIdUser); | ||
} | ||
|
||
return familyOpenCGAResult; | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Reminder : We'll need to add the
@Category
to this test once the categories are merged