Merge branch '161-new-project-space-for-rest-api' into 'master'

Resolve "New project space for REST API" Closes #161, #145, #149, #162, #160, #151, #34, #33, #32, #165, and #150 See merge request bright-giant/sirius/sirius-frontend!41
sirius-ms · Apr 15, 2024 · 9602768 · 9602768
2 parents f4b8bc6 + b85d77e
commit 9602768
Show file tree

Hide file tree

Showing 161 changed files with 8,099 additions and 3,444 deletions.
diff --git a/sirius_cli/build.gradle b/sirius_cli/build.gradle
@@ -41,7 +41,9 @@ dependencies {
     api group: "de.unijena.bioinf.ms", name: "sirius_web_client", version: "$fingeridVersion"
     api group: "de.unijena.bioinf.ms", name: "fingerid_project_space_oss", version: "$fingeridVersion"
     api group: "de.unijena.bioinf.ms", name: "document-storage", version: "$siriusVersion"
-//    api group: "de.unijena.bioinf.ms", name: "ms_persistence_model_oss", version: "$siriusVersion"
+    api group: "de.unijena.bioinf.ms", name: "ms_persistence_model_oss", version: "$siriusVersion"
+    api group: "de.unijena.bioinf.ms", name: "ms_persistence_document_storage_oss", version: "$siriusVersion"
+    api group: "de.unijena.bioinf.ms", name: "lcms2", version: "$siriusVersion"
 
     api group: "de.unijena.bioinf.ms", name: "FTAlignmentTool", version: "$siriusVersion"
 

diff --git a/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/SiriusCLIApplication.java b/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/SiriusCLIApplication.java
@@ -31,6 +31,7 @@
 import de.unijena.bioinf.ms.frontend.workflow.WorkFlowSupplier;
 import de.unijena.bioinf.ms.frontend.workflow.WorkflowBuilder;
 import de.unijena.bioinf.ms.properties.PropertyManager;
+import de.unijena.bioinf.projectspace.NitriteProjectSpaceManagerFactory;
 import de.unijena.bioinf.projectspace.SiriusProjectSpaceManagerFactory;
 import de.unijena.bioinf.rest.ProxyManager;
 import org.jetbrains.annotations.NotNull;
@@ -54,13 +55,14 @@ public class SiriusCLIApplication {
     public static void main(String[] args) {
         runMain(args, List.of());
     }
+
     public static void runMain(String[] args, List<StandaloneTool<?>> injectTools) {
         System.setProperty(APP_TYPE_PROPERTY_KEY, "CLI");
         {
             List<String> argsl = List.of(args);
             int i = argsl.indexOf("--workspace");
             if (i >= 0)
-                System.setProperty("de.unijena.bioinf.sirius.ws.location", args[i+1].replace("'","").replace("\"",""));
+                System.setProperty("de.unijena.bioinf.sirius.ws.location", args[i + 1].replace("'", "").replace("\"", ""));
         }
         if (TIME)
             t1 = System.currentTimeMillis();
@@ -73,7 +75,10 @@ public static void runMain(String[] args, List<StandaloneTool<?>> injectTools) {
             configureShutDownHook(shutdownWebservice());
             measureTime("Start Run method");
             run(args, () -> new WorkflowBuilder(
-                    new CLIRootOptions(new DefaultParameterConfigLoader(), new SiriusProjectSpaceManagerFactory()), injectTools));
+                    PropertyManager.getProperty("sirius.middleware.project-space", null, "NITRITE-NOSQL").equals("NITRITE-NOSQL")
+                            ? new CLIRootOptions(new DefaultParameterConfigLoader(), new NitriteProjectSpaceManagerFactory())
+                            : new CLIRootOptions(new DefaultParameterConfigLoader(), new SiriusProjectSpaceManagerFactory())
+                    , injectTools));
         } finally {
             System.exit(0);
         }
@@ -107,7 +112,7 @@ public static void configureShutDownHook(@NotNull final Runnable... additionalAc
                         Files.deleteIfExists(ApplicationCore.TOKEN_FILE);
                 } catch (IOException e) {
                     e.printStackTrace();
-                }finally {
+                } finally {
                     ProxyManager.disconnect();
                     if (successfulParsed && PropertyManager.DEFAULTS.createInstanceWithDefaults(PrintCitations.class).value)
                         ApplicationCore.BIBTEX.citeToSystemErr();

diff --git a/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/CLIRootOptions.java b/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/CLIRootOptions.java
@@ -182,7 +182,7 @@ protected ProjectSpaceManager compute() throws Exception {
                 InputFilesOptions input = getInput();
                 if (space != null) {
                     if (input != null)
-                        SiriusJobs.getGlobalJobManager().submitJob(new InstanceImporter(space, (exp) -> exp.getIonMass() < maxMz, (c) -> c.getIonMass().map(m -> m < maxMz).orElse(true)).makeImportJJob(input)).awaitResult();
+                        SiriusJobs.getGlobalJobManager().submitJob(new InstanceImporter(space, (exp) -> exp.getIonMass() < maxMz).makeImportJJob(input)).awaitResult();
                     if (space.size() < 1)
                         logInfo("No Input has been imported to Project-Space. Starting application without input data.");
                     return space;

diff --git a/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/DataSetJob.java b/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/DataSetJob.java
@@ -31,7 +31,6 @@
 import java.util.Map;
 import java.util.function.Consumer;
 import java.util.function.Function;
-import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 public abstract class DataSetJob extends ToolChainJobImpl<Iterable<Instance>> implements ToolChainJob<Iterable<Instance>> {
@@ -41,15 +40,12 @@ public abstract class DataSetJob extends ToolChainJobImpl<Iterable<Instance>> im
 
     protected long maxProgress = 100;
 
-    private final Predicate<Instance> inputValidator;
-
-    public DataSetJob(@NotNull Predicate<Instance> inputValidator, @NotNull JobSubmitter submitter) {
-        this(inputValidator, submitter, ReqJobFailBehaviour.WARN);
+    public DataSetJob(@NotNull JobSubmitter submitter) {
+        this(submitter, ReqJobFailBehaviour.WARN);
     }
 
-    public DataSetJob(@NotNull Predicate<Instance> inputValidator, @NotNull JobSubmitter submitter, @NotNull ReqJobFailBehaviour failBehaviour) {
+    public DataSetJob(@NotNull JobSubmitter submitter, @NotNull ReqJobFailBehaviour failBehaviour) {
         super(submitter, failBehaviour);
-        this.inputValidator = inputValidator;
     }
 
     @Override
@@ -58,9 +54,8 @@ protected Iterable<Instance> compute() throws Exception {
         maxProgress = inputInstances.size() * 101L + 1;
         updateProgress(0L, maxProgress, Math.round(.25 * inputInstances.size()), "Invalidate existing Results and Recompute!");
 
-        //todo maybe make decidable if any or all match
         final boolean hasResults = inputInstances.stream().anyMatch(this::isAlreadyComputed);
-        final boolean recompute = inputInstances.stream().anyMatch(this::isRecompute);
+        final boolean recompute = inputInstances.stream().anyMatch(Instance::isRecompute);
 
         updateProgress(Math.round(.5 * inputInstances.size()), "Invalidate existing Results and Recompute!");
 
@@ -79,7 +74,7 @@ protected Iterable<Instance> compute() throws Exception {
             updateProgress(Math.round(.9 * inputInstances.size()), "Invalidate existing Results and Recompute!");
 
             progressInfo( "Start computation...");
-            inputInstances.forEach(this::enableRecompute); // enable recompute so that following tools will recompute if results exist.
+            inputInstances.forEach(Instance::enableRecompute); // enable recompute so that following tools will recompute if results exist.
             updateProgress(inputInstances.size());
             computeAndAnnotateResult(inputInstances);
             updateProgress(maxProgress - 1, "DONE!");
@@ -92,7 +87,7 @@ protected Iterable<Instance> compute() throws Exception {
 
     protected void checkInputs() {
         {
-            final Map<Boolean, List<Instance>> splitted = inputInstances.stream().collect(Collectors.partitioningBy(inputValidator));
+            final Map<Boolean, List<Instance>> splitted = inputInstances.stream().collect(Collectors.partitioningBy(this::isInstanceValid));
             inputInstances = splitted.get(true);
             failedInstances = splitted.get(false);
         }
@@ -133,6 +128,7 @@ public synchronized void handleFinishedRequiredJob(JJob required) {
         }
     }
 
+    protected abstract boolean isInstanceValid(Instance instance);
 
     protected abstract void computeAndAnnotateResult(final @NotNull List<Instance> expRes) throws Exception;
 

diff --git a/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/InstanceJob.java b/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/InstanceJob.java
@@ -51,9 +51,9 @@ public InstanceJob(JobSubmitter submitter) {
     @Override
     public synchronized void handleFinishedRequiredJob(JJob required) {
         final Object r = required.result();
-        if (r instanceof Instance)
+        if (r instanceof Instance inst)
             if (input == null || input.equals(r))
-                input = (Instance) r;
+                input = inst;
     }
 
 
@@ -68,21 +68,20 @@ protected Instance compute() throws Exception {
         updateProgress(1);
 
         checkForInterruption();
-        if (!hasResults || isRecompute(input)) {
-            if (hasResults){
+        if (!hasResults || input.isRecompute()) {
+            if (hasResults) {
                 invalidateResults(input);
             }
             updateProgress(2, "Invalidate existing Results and Recompute!");
             progressInfo("Start computation...");
-            setRecompute(input,true); // enable recompute so that following tools will recompute if results exist.
+            input.setRecompute(true); // enable recompute so that following tools will recompute if results exist.
             checkForInterruption();
             computeAndAnnotateResult(input);
             checkForInterruption();
-            updateProgress(JobProgressEvent.DEFAULT_MAX- 1, "DONE!");
+            updateProgress(JobProgressEvent.DEFAULT_MAX - 1, "DONE!");
         } else {
-            updateProgress(JobProgressEvent.DEFAULT_MAX- 1, "Skipping Job because results already Exist and recompute not requested.");
+            updateProgress(JobProgressEvent.DEFAULT_MAX - 1, "Skipping Job because results already Exist and recompute not requested.");
         }
-
         return input;
     }
 
@@ -103,14 +102,15 @@ public String getProjectName() {
 
     /**
      * Check if the input is valid for computation. May be overwritten by implementations for additional checks.
+     *
      * @return false if input data is fine and true if data should be skipped gently. IllegalArgumentException is thrown
      * if the input check needs to cause job failure
      */
     protected boolean checkInput() {
         if (input == null)
             throw new IllegalArgumentException("No Input available! Maybe a previous job could not provide the needed results due to failure.");
         if (needsMs2())
-            if (input.getExperiment().getMs2Spectra().isEmpty()){
+            if (input.getExperiment().getMs2Spectra().isEmpty()) {
                 logInfo("Input contains no non empty MS/MS spectrum but MS/MS data is mandatory for this job. Skipping Instance!");
                 return true;
             }
@@ -142,9 +142,12 @@ protected void checkFingerprintCompatibilityOrThrow() throws TimeoutException, I
         if (!checkFingerprintCompatibility())
             throw new IncompatibleFingerprintDataException();
     }
+
     protected boolean checkFingerprintCompatibility() throws TimeoutException, InterruptedException {
         return input.getProjectSpaceManager().checkAndFixDataFiles(this::checkForInterruption);
     }
 
-    protected boolean needsMs2(){return true;};
+    protected boolean needsMs2() {
+        return true;
+    }
 }
diff --git a/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/ToolChainJob.java b/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/ToolChainJob.java
@@ -21,7 +21,6 @@
 
 import de.unijena.bioinf.jjobs.JobSubmitter;
 import de.unijena.bioinf.jjobs.ProgressJJob;
-import de.unijena.bioinf.ms.annotations.RecomputeResults;
 import de.unijena.bioinf.projectspace.Instance;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -30,23 +29,6 @@
 import java.util.function.Function;
 
 public interface ToolChainJob<T> extends ProgressJJob<T> {
-
-    default boolean isRecompute(final @NotNull Instance inst) {
-        return inst.getExperiment().getAnnotation(RecomputeResults.class, () -> RecomputeResults.FALSE).value;
-    }
-
-    default boolean enableRecompute(final @NotNull Instance inst) {
-        return setRecompute(inst, true);
-    }
-
-    default boolean disableRecompute(final @NotNull Instance inst) {
-        return setRecompute(inst, false);
-    }
-
-    default boolean setRecompute(final @NotNull Instance inst, boolean recompute) {
-        return inst.getExperiment().setAnnotation(RecomputeResults.class, RecomputeResults.newInstance(recompute));
-    }
-
     boolean isAlreadyComputed(final @NotNull Instance inst);
 
     default String getToolName() {

diff --git a/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/canopus/CanopusOptions.java b/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/canopus/CanopusOptions.java
@@ -19,7 +19,6 @@
 
 package de.unijena.bioinf.ms.frontend.subtools.canopus;
 
-import de.unijena.bioinf.canopus.CanopusResult;
 import de.unijena.bioinf.ms.frontend.subtools.InstanceJob;
 import de.unijena.bioinf.ms.frontend.subtools.Provide;
 import de.unijena.bioinf.ms.frontend.subtools.ToolChainOptions;
@@ -57,7 +56,7 @@ public InstanceJob.Factory<CanopusSubToolJob> call() throws Exception {
 
     @Override
     public Consumer<Instance> getInvalidator() {
-        return inst -> inst.deleteFromFormulaResults(CanopusResult.class);
+        return Instance::deleteCanopusResult;
     }
 
     @Override

diff --git a/...s_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/canopus/CanopusSubToolJob.java b/...s_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/canopus/CanopusSubToolJob.java
@@ -19,8 +19,6 @@
 
 package de.unijena.bioinf.ms.frontend.subtools.canopus;
 
-import de.unijena.bioinf.ChemistryBase.algorithm.scoring.FormulaScore;
-import de.unijena.bioinf.ChemistryBase.algorithm.scoring.SScored;
 import de.unijena.bioinf.ChemistryBase.ms.utils.Spectrums;
 import de.unijena.bioinf.canopus.CanopusResult;
 import de.unijena.bioinf.fingerid.FingerprintResult;
@@ -30,10 +28,8 @@
 import de.unijena.bioinf.ms.frontend.utils.PicoUtils;
 import de.unijena.bioinf.ms.rest.model.canopus.CanopusJobInput;
 import de.unijena.bioinf.ms.webapi.WebJJob;
-import de.unijena.bioinf.projectspace.FormulaResult;
-import de.unijena.bioinf.projectspace.FormulaScoring;
+import de.unijena.bioinf.projectspace.FCandidate;
 import de.unijena.bioinf.projectspace.Instance;
-import de.unijena.bioinf.projectspace.ProjectSpaceManagers;
 import de.unijena.bioinf.rest.NetUtils;
 import org.jetbrains.annotations.NotNull;
 
@@ -43,29 +39,28 @@
 import java.util.stream.Collectors;
 
 public class CanopusSubToolJob extends InstanceJob {
-    private Map<FormulaResult, WebJJob<CanopusJobInput, ?, CanopusResult, ?>> jobs;
+    private Map<FCandidate<?>, WebJJob<CanopusJobInput, ?, CanopusResult, ?>> jobs;
     public CanopusSubToolJob(JobSubmitter submitter) {
         super(submitter);
         asWEBSERVICE();
     }
 
     @Override
     public boolean isAlreadyComputed(@NotNull Instance inst) {
-        return inst.loadCompoundContainer().hasResults() && inst.loadFormulaResults(CanopusResult.class).stream().anyMatch((it -> it.getCandidate().hasAnnotation(CanopusResult.class)));
+        return inst.hasCanopusResult();
     }
 
     @Override
     protected void computeAndAnnotateResult(final @NotNull Instance inst) throws Exception {
-        List<? extends SScored<FormulaResult, ? extends FormulaScore>> input = inst.loadFormulaResults(FormulaScoring.class, FingerprintResult.class, CanopusResult.class);
+        List<FCandidate<?>> inputData = inst.getCanopusInput().stream()
+                .filter(c -> c.hasAnnotation(FingerprintResult.class))
+                .toList();
 
         checkForInterruption();
 
-        // create input
-        List<FormulaResult> res = input.stream().map(SScored::getCandidate)
-                .filter(ir -> ir.hasAnnotation(FingerprintResult.class)).toList();
 
         // check for valid input
-        if (res.isEmpty()) {
+        if (inputData.isEmpty()) {
             logInfo("Skipping because there are no formula results available");
             return;
         }
@@ -75,7 +70,7 @@ protected void computeAndAnnotateResult(final @NotNull Instance inst) throws Exc
         updateProgress(10);
         checkForInterruption();
 
-        NetUtils.tryAndWait(() -> ProjectSpaceManagers.writeCanopusDataIfMissing(inst.getProjectSpaceManager(), ApplicationCore.WEB_API), this::checkForInterruption);
+        NetUtils.tryAndWait(() -> inst.getProjectSpaceManager().writeCanopusDataIfMissing(ApplicationCore.WEB_API), this::checkForInterruption);
 
         updateProgress(20);
         checkForInterruption();
@@ -84,7 +79,7 @@ protected void computeAndAnnotateResult(final @NotNull Instance inst) throws Exc
         updateProgress(25);
 
         // submit canopus jobs for Identification results that contain CSI:FingerID results
-        jobs = res.stream().collect(Collectors.toMap(r -> r, ir -> buildAndSubmitRemote(ir, specHash)));
+        jobs = inputData.stream().collect(Collectors.toMap(r -> r, ir -> buildAndSubmitRemote(ir, specHash)));
         updateProgress(30);
 
 
@@ -93,15 +88,14 @@ protected void computeAndAnnotateResult(final @NotNull Instance inst) throws Exc
         updateProgress(80);
 
         // write canopus results
-        for (FormulaResult r : res)
-            inst.updateFormulaResult(r, CanopusResult.class);
+        inst.saveCanopusResult(inputData);
         updateProgress(97);
     }
 
-    private WebJJob<CanopusJobInput, ?, CanopusResult, ?> buildAndSubmitRemote(@NotNull final FormulaResult ir, int specHash)  {
+    private WebJJob<CanopusJobInput, ?, CanopusResult, ?> buildAndSubmitRemote(@NotNull final FCandidate<?> ir, int specHash)  {
         try {
             return ApplicationCore.WEB_API.submitCanopusJob(
-                    ir.getId().getMolecularFormula(), ir.getId().getIonType().getCharge(),
+                    ir.getMolecularFormula(), ir.getAdduct().getCharge(),
                     ir.getAnnotationOrThrow(FingerprintResult.class).fingerprint, specHash
             );
         } catch (IOException e) {