storage: Add version to FamilyIndex status #1901

opencb · Mar 17, 2022 · d1ba365 · d1ba365
1 parent 68076e7
commit d1ba365
Show file tree

Hide file tree

Showing 24 changed files with 447 additions and 259 deletions.
diff --git a/...ysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java b/...ysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java
@@ -69,6 +69,7 @@
 import org.opencb.opencga.core.models.study.StudyAclEntry;
 import org.opencb.opencga.core.response.OpenCGAResult;
 import org.opencb.opencga.core.response.VariantQueryResult;
+import org.opencb.opencga.core.tools.ToolParams;
 import org.opencb.opencga.storage.core.StorageEngineFactory;
 import org.opencb.opencga.storage.core.StoragePipelineResult;
 import org.opencb.opencga.storage.core.exceptions.StorageEngineException;
@@ -383,6 +384,14 @@ public void sampleIndexAnnotate(String study, List<String> samples, ObjectMap pa
         });
     }
 
+    public DataResult<List<String>> familyIndexUpdate(String study,
+                                                ObjectMap params, String token)
+            throws CatalogException, StorageEngineException {
+        return secureOperation(VariantFamilyIndexOperationTool.ID, study, params, token, engine -> {
+            return engine.familyIndexUpdate(study, params);
+        });
+    }
+
     public DataResult<List<String>> familyIndex(String study, List<String> familiesStr, boolean skipIncompleteFamilies,
                                                 ObjectMap params, String token)
             throws CatalogException, StorageEngineException {
@@ -403,7 +412,6 @@ public DataResult<List<String>> familyIndex(String study, List<String> familiesS
                     trios.addAll(catalogUtils.getTriosFromFamily(study, family, metadataManager, skipIncompleteFamilies, token));
                 }
             }
-
             return engine.familyIndex(study, trios, params);
         });
     }
@@ -505,10 +513,14 @@ public OpenCGAResult<Job> configureSampleIndex(String studyStr, SampleIndexConfi
                 return new OpenCGAResult<>(0, new ArrayList<>(), 0, new ArrayList<>(), 0);
             } else {
                 // If changes, launch sample-index-run
-                VariantSampleIndexParams params =
+                ToolParams params =
                         new VariantSampleIndexParams(Collections.singletonList(ParamConstants.ALL), true, true, false);
-                return catalogManager.getJobManager().submit(studyFqn, VariantSampleIndexOperationTool.ID, null,
-                        params.toParams(STUDY_PARAM, studyFqn), token);
+                Job job = catalogManager.getJobManager().submit(studyFqn, VariantSampleIndexOperationTool.ID, null,
+                        params.toParams(STUDY_PARAM, studyFqn), token).first();
+                params = new VariantFamilyIndexParams(Collections.emptyList(), false, true, false);
+                Job job2 = catalogManager.getJobManager().submit(studyFqn, VariantFamilyIndexOperationTool.ID, null,
+                        params.toParams(STUDY_PARAM, studyFqn), token).first();
+                return new OpenCGAResult<>(0, new ArrayList<>(), 2, Arrays.asList(job, job2), 0);
             }
         });
     }

diff --git a/.../java/org/opencb/opencga/analysis/variant/operations/VariantFamilyIndexOperationTool.java b/.../java/org/opencb/opencga/analysis/variant/operations/VariantFamilyIndexOperationTool.java
@@ -19,9 +19,11 @@
 import org.apache.commons.collections4.CollectionUtils;
 import org.opencb.commons.datastore.core.DataResult;
 import org.opencb.commons.datastore.core.Event;
-import org.opencb.opencga.core.tools.annotations.Tool;
-import org.opencb.opencga.core.models.operations.variant.VariantFamilyIndexParams;
+import org.opencb.opencga.core.api.ParamConstants;
 import org.opencb.opencga.core.models.common.Enums;
+import org.opencb.opencga.core.models.operations.variant.VariantFamilyIndexParams;
+import org.opencb.opencga.core.tools.annotations.Tool;
+import org.opencb.opencga.core.tools.annotations.ToolParams;
 
 import java.util.List;
 
@@ -33,29 +35,43 @@ public class VariantFamilyIndexOperationTool extends OperationTool {
     public static final String DESCRIPTION = "Build the family index";
 
     private String study;
-    private VariantFamilyIndexParams variantFamilyIndexParams;
+
+    @ToolParams
+    protected VariantFamilyIndexParams variantFamilyIndexParams;
 
     @Override
     protected void check() throws Exception {
         super.check();
 
-        variantFamilyIndexParams = VariantFamilyIndexParams.fromParams(VariantFamilyIndexParams.class, params);
         study = getStudyFqn();
 
-        if (CollectionUtils.isEmpty(variantFamilyIndexParams.getFamily())) {
-            throw new IllegalArgumentException("Empty list of families");
+        List<String> list = variantFamilyIndexParams.getFamily();
+        if (variantFamilyIndexParams.isUpdateIndex()) {
+            if (list.size() > 1 || list.size() == 1 && !list.get(0).equals(ParamConstants.ALL)) {
+                throw new IllegalArgumentException("Unaccepted parameter \"family\" when updating index.");
+            }
+        } else {
+            if (CollectionUtils.isEmpty(list)) {
+                throw new IllegalArgumentException("Empty list of families");
+            }
         }
+
     }
 
     @Override
     protected void run() throws Exception {
         step(() -> {
-            DataResult<List<String>> trios = variantStorageManager.familyIndex(
-                    study,
-                    variantFamilyIndexParams.getFamily(),
-                    variantFamilyIndexParams.isSkipIncompleteFamilies(),
-                    params,
-                    token);
+            DataResult<List<String>> trios;
+            if (variantFamilyIndexParams.isUpdateIndex()) {
+                trios = variantStorageManager.familyIndexUpdate(study, params, token);
+            } else {
+                trios = variantStorageManager.familyIndex(
+                        study,
+                        variantFamilyIndexParams.getFamily(),
+                        variantFamilyIndexParams.isSkipIncompleteFamilies(),
+                        params,
+                        token);
+            }
             if (trios.getEvents() != null) {
                 for (Event event : trios.getEvents()) {
                     addEvent(event.getType(), event.getMessage());

diff --git a/...in/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java b/...in/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java
@@ -514,6 +514,7 @@ private void familyIndex()
         VariantFamilyIndexParams params = new VariantFamilyIndexParams(
                 cliOptions.family,
                 cliOptions.overwrite,
+                cliOptions.update,
                 cliOptions.skipIncompleteFamilies);
 
         toolRunner.execute(VariantFamilyIndexOperationTool.class,

diff --git a/...-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java b/...-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java
@@ -539,6 +539,9 @@ public class FamilyIndexCommandOptions extends GeneralCliOptions.StudyOption {
         @Parameter(names = {"--overwrite"}, description = "Overwrite existing values")
         public boolean overwrite = false;
 
+        @Parameter(names = {"--update"}, description = "Update family index")
+        public boolean update = false;
+
         @Parameter(names = {"--skip-incomplete-families"}, description = "Do not process incomplete families.")
         public boolean skipIncompleteFamilies = false;
 

diff --git a/.../java/org/opencb/opencga/app/cli/main/executors/operations/OperationsCommandExecutor.java b/.../java/org/opencb/opencga/app/cli/main/executors/operations/OperationsCommandExecutor.java
@@ -329,6 +329,7 @@ private RestResponse<Job> variantFamilyIndex() throws ClientException {
                 new VariantFamilyIndexParams(
                         cliOptions.family,
                         cliOptions.overwrite,
+                        cliOptions.update,
                         cliOptions.skipIncompleteFamilies
                 ), params);
     }

diff --git a/...a-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsCommandOptions.java b/...a-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsCommandOptions.java
@@ -461,6 +461,9 @@ public class VariantFamilyGenotypeIndexCommandOptions extends GeneralCliOptions.
         @Parameter(names = {"--overwrite"}, description = "Overwrite existing values")
         public boolean overwrite = false;
 
+        @Parameter(names = {"--update"}, description = "Update family index")
+        public boolean update = false;
+
         @Parameter(names = {"--skip-incomplete-families"}, description = "Do not process incomplete families.")
         public boolean skipIncompleteFamilies = false;
     }

diff --git a/...c/main/java/org/opencb/opencga/app/migrations/v2_2_0/storage/UpdateSampleIndexStatus.java b/...c/main/java/org/opencb/opencga/app/migrations/v2_2_0/storage/UpdateSampleIndexStatus.java
@@ -26,6 +26,8 @@ public class UpdateSampleIndexStatus extends StorageMigrationTool {
     protected static final String SAMPLE_INDEX_ANNOTATION_STATUS = "sampleIndexAnnotation";
     protected static final String SAMPLE_INDEX_ANNOTATION_VERSION = "sampleIndexAnnotationVersion";
 
+    private static final String FAMILY_INDEX_STATUS = "family_index";
+
     private final String markKey = "migration_" + getAnnotation().id();
     private final int markValue = getAnnotation().patch();
 
@@ -66,6 +68,7 @@ protected void updateSampleMetadata(SampleMetadata sampleMetadata) {
             addVersionToSampleIndexStatus(sampleMetadata);
             renameOldSampleIndexAnnotationStatus(sampleMetadata);
             addVersionToSampleIndexAnnotationStatus(sampleMetadata);
+            addVersionToFamilyIndexStatus(sampleMetadata);
 //        removeOldSampleIndexStatus(sampleMetadata);
             addUpdatedMark(sampleMetadata);
         }
@@ -109,10 +112,17 @@ protected void addVersionToSampleIndexAnnotationStatus(SampleMetadata sampleMeta
         }
     }
 
+    private void addVersionToFamilyIndexStatus(SampleMetadata sampleMetadata) {
+        if (sampleMetadata.getStatus(FAMILY_INDEX_STATUS) == TaskMetadata.Status.READY) {
+            sampleMetadata.setFamilyIndexStatus(TaskMetadata.Status.READY, StudyMetadata.DEFAULT_SAMPLE_INDEX_VERSION);
+        }
+    }
+
     private void removeOldSampleIndexStatus(SampleMetadata sampleMetadata) {
         sampleMetadata.getStatus().remove(SAMPLE_INDEX_STATUS);
         sampleMetadata.getStatus().remove(SAMPLE_INDEX_ANNOTATION_STATUS_OLD);
         sampleMetadata.getStatus().remove(SAMPLE_INDEX_ANNOTATION_STATUS);
+        sampleMetadata.getStatus().remove(FAMILY_INDEX_STATUS);
     }
 
 

diff --git a/...main/java/org/opencb/opencga/core/models/operations/variant/VariantFamilyIndexParams.java b/...main/java/org/opencb/opencga/core/models/operations/variant/VariantFamilyIndexParams.java
@@ -25,14 +25,16 @@ public class VariantFamilyIndexParams extends ToolParams {
     public static final String DESCRIPTION = "Variant family index params.";
     private List<String> family;
     private boolean overwrite;
+    private boolean updateIndex;
     private boolean skipIncompleteFamilies;
 
     public VariantFamilyIndexParams() {
     }
 
-    public VariantFamilyIndexParams(List<String> family, boolean overwrite, boolean skipIncompleteFamilies) {
+    public VariantFamilyIndexParams(List<String> family, boolean overwrite, boolean updateIndex, boolean skipIncompleteFamilies) {
         this.family = family;
         this.overwrite = overwrite;
+        this.updateIndex = updateIndex;
         this.skipIncompleteFamilies = skipIncompleteFamilies;
     }
 
@@ -45,6 +47,15 @@ public VariantFamilyIndexParams setFamily(List<String> family) {
         return this;
     }
 
+    public boolean isUpdateIndex() {
+        return updateIndex;
+    }
+
+    public VariantFamilyIndexParams setUpdateIndex(boolean updateIndex) {
+        this.updateIndex = updateIndex;
+        return this;
+    }
+
     public boolean isOverwrite() {
         return overwrite;
     }

diff --git a/...src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java b/...src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java
@@ -841,6 +841,10 @@ public <E extends Exception> SampleMetadata updateSampleMetadata(int studyId, in
         }
     }
 
+    public Iterable<SampleMetadata> sampleMetadataIterable(int studyId) {
+        return () -> sampleMetadataIterator(studyId);
+    }
+
     public Iterator<SampleMetadata> sampleMetadataIterator(int studyId) {
         return sampleDBAdaptor.sampleMetadataIterator(studyId);
     }