diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/BeanNames.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/BeanNames.java index 26c704e7e..927c29213 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/BeanNames.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/BeanNames.java @@ -55,6 +55,8 @@ public class BeanNames { public static final String PROGRESS_LISTENER = "PROGRESS_LISTENER"; + public static final String ACCESSIONING_SHUTDOWN_STEP = "ACCESSIONING_SHUTDOWN_STEP"; + public static final String CLUSTERING_FROM_VCF_STEP = "CLUSTERING_FROM_VCF_STEP"; public static final String CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP = "CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP"; diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java index df7c621c3..b9517c10a 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java @@ -24,6 +24,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.ACCESSIONING_SHUTDOWN_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTER_UNCLUSTERED_VARIANTS_JOB; @@ -42,6 +43,7 @@ public Job clusteringFromMongoJob( @Qualifier(PROCESS_RS_SPLIT_CANDIDATES_STEP) Step processRSSplitCandidatesStep, @Qualifier(CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP) Step clearRSMergeAndSplitCandidatesStep, @Qualifier(CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP) Step clusteringNonClusteredVariantsFromMongoStep, + @Qualifier(ACCESSIONING_SHUTDOWN_STEP) Step accessioningShutdownStep, JobBuilderFactory jobBuilderFactory) { return jobBuilderFactory.get(CLUSTER_UNCLUSTERED_VARIANTS_JOB) .incrementer(new RunIdIncrementer()) @@ -49,6 +51,7 @@ public Job clusteringFromMongoJob( .next(processRSSplitCandidatesStep) .next(clearRSMergeAndSplitCandidatesStep) .next(clusteringNonClusteredVariantsFromMongoStep) + .next(accessioningShutdownStep) .build(); } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java index 16a422fcd..786900cd2 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java @@ -40,6 +40,7 @@ import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.ACCESSIONING_SHUTDOWN_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_NEW_RS_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP; @@ -79,6 +80,7 @@ public Job clusteringFromMongoJob(@Qualifier(CLUSTERING_CLUSTERED_VARIANTS_FROM_ @Qualifier(PROCESS_RS_SPLIT_CANDIDATES_STEP) Step processRSSplitCandidatesStep, @Qualifier(CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP) Step clearRSMergeAndSplitCandidatesStep, @Qualifier(CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP) Step clusteringNonClusteredVariantsFromMongoStep, + @Qualifier(ACCESSIONING_SHUTDOWN_STEP) Step accessioningShutdownStep, // Back-propagate RS that were newly created in the remapped assembly @Qualifier(BACK_PROPAGATE_NEW_RS_STEP) Step backPropagateNewRSStep, // Back-propagate RS in the remapped assembly that were split or merged @@ -101,12 +103,14 @@ public Job clusteringFromMongoJob(@Qualifier(CLUSTERING_CLUSTERED_VARIANTS_FROM_ .next(processRSSplitCandidatesStep) .next(clearRSMergeAndSplitCandidatesStep) .next(clusteringNonClusteredVariantsFromMongoStep) + .next(accessioningShutdownStep) .next(backPropagateNewRSStep) .next(backPropagateSplitMergedRSStep).build()) .on("*").end() .from(jobExecutionDecider) .on("FALSE") .to(clusteringNonClusteredVariantsFromMongoStep) + .next(accessioningShutdownStep) .on("*").end() .end().build(); } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java index 5d78ca2ec..4b4262503 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java @@ -24,6 +24,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.ACCESSIONING_SHUTDOWN_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.STUDY_CLUSTERING_JOB; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.STUDY_CLUSTERING_STEP; @@ -33,10 +34,12 @@ public class StudyClusteringJobConfiguration { @Bean(STUDY_CLUSTERING_JOB) public Job studyClusteringJob(@Qualifier(STUDY_CLUSTERING_STEP) Step clusteringStep, + @Qualifier(ACCESSIONING_SHUTDOWN_STEP) Step accessioningShutdownStep, JobBuilderFactory jobBuilderFactory) { return jobBuilderFactory.get(STUDY_CLUSTERING_JOB) .incrementer(new RunIdIncrementer()) .start(clusteringStep) + .next(accessioningShutdownStep) .build(); } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/AccessioningShutdownStepConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/AccessioningShutdownStepConfiguration.java new file mode 100644 index 000000000..1c67b1b0e --- /dev/null +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/AccessioningShutdownStepConfiguration.java @@ -0,0 +1,33 @@ +package uk.ac.ebi.eva.accession.clustering.configuration.batch.steps; + +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import uk.ac.ebi.eva.accession.core.service.nonhuman.ClusteredVariantAccessioningService; +import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; + +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.ACCESSIONING_SHUTDOWN_STEP; + +@Configuration +@EnableBatchProcessing +public class AccessioningShutdownStepConfiguration { + @Autowired + private SubmittedVariantAccessioningService submittedVariantAccessioningService; + + @Autowired + private ClusteredVariantAccessioningService clusteredVariantAccessioningService; + + @Bean(ACCESSIONING_SHUTDOWN_STEP) + public Step accessioningShutDownStep(StepBuilderFactory stepBuilderFactory) { + return stepBuilderFactory.get(ACCESSIONING_SHUTDOWN_STEP) + .tasklet((contribution, chunkContext) -> { + submittedVariantAccessioningService.shutDownAccessionGenerator(); + clusteredVariantAccessioningService.shutDownAccessionGenerator(); + return null; + }) + .build(); + } +} \ No newline at end of file diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java index ad2c6cce6..e34c4bebe 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java @@ -64,6 +64,7 @@ import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.ACCESSIONING_SHUTDOWN_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_NEW_RS_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP; @@ -160,6 +161,7 @@ public void jobFromMongo() throws Exception { expectedSteps.add(PROCESS_RS_SPLIT_CANDIDATES_STEP); expectedSteps.add(CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP); expectedSteps.add(CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP); + expectedSteps.add(ACCESSIONING_SHUTDOWN_STEP); expectedSteps.add(BACK_PROPAGATE_NEW_RS_STEP); expectedSteps.add(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP); assertStepsExecuted(expectedSteps, jobExecution); @@ -171,7 +173,9 @@ public void jobFromMongo() throws Exception { @UsingDataSet(locations = {"/test-data/submittedVariantEntityStudyReader.json"}) public void studyJobFromMongo() throws Exception { JobExecution jobExecution = jobLauncherTestUtilsStudyFromMongo.launchJob(); - List expectedSteps = Collections.singletonList(STUDY_CLUSTERING_STEP); + List expectedSteps = new ArrayList<>(); + expectedSteps.add(STUDY_CLUSTERING_STEP); + expectedSteps.add(ACCESSIONING_SHUTDOWN_STEP); assertStepsExecuted(expectedSteps, jobExecution); assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); } diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java index d9c75a056..29a0d76ee 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java @@ -40,6 +40,7 @@ import uk.ac.ebi.eva.accession.clustering.configuration.batch.listeners.ListenersConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.policies.ChunkSizeCompletionPolicyConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.processors.ClusteringVariantProcessorConfiguration; +import uk.ac.ebi.eva.accession.clustering.configuration.batch.steps.AccessioningShutdownStepConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.steps.ClusteringFromMongoStepConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.steps.ClusteringFromVcfStepConfiguration; import uk.ac.ebi.eva.accession.clustering.runner.ClusteringCommandLineRunner; @@ -71,7 +72,8 @@ BackPropagatedRSWriterConfiguration.class, ListenersConfiguration.class, ClusteringCommandLineRunner.class, - ChunkSizeCompletionPolicyConfiguration.class}) + ChunkSizeCompletionPolicyConfiguration.class, + AccessioningShutdownStepConfiguration.class}) public class BatchTestConfiguration { public static final String JOB_LAUNCHER_FROM_VCF = "JOB_LAUNCHER_FROM_VCF"; diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/generators/DbsnpMonotonicAccessionGenerator.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/generators/DbsnpMonotonicAccessionGenerator.java index cfdf9713b..a6d5bff6b 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/generators/DbsnpMonotonicAccessionGenerator.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/generators/DbsnpMonotonicAccessionGenerator.java @@ -30,7 +30,7 @@ public class DbsnpMonotonicAccessionGenerator extends MonotonicAccessionG public DbsnpMonotonicAccessionGenerator(String categoryId, String applicationInstanceId, ContiguousIdBlockService contiguousIdBlockService) { - super(categoryId, applicationInstanceId, contiguousIdBlockService, (long[])null); + super(categoryId, applicationInstanceId, contiguousIdBlockService, null); } @Override diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/service/nonhuman/ClusteredVariantAccessioningService.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/service/nonhuman/ClusteredVariantAccessioningService.java index 93a5bf20e..cde6aad1f 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/service/nonhuman/ClusteredVariantAccessioningService.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/service/nonhuman/ClusteredVariantAccessioningService.java @@ -243,4 +243,9 @@ public AccessionWrapper getLastInactive(Long ac } } + public void shutDownAccessionGenerator(){ + accessioningService.shutDownAccessioning(); + accessioningServiceDbsnp.shutDownAccessioning(); + } + } diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/listeners/SubsnpAccessionsStepListener.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/listeners/SubsnpAccessionsStepListener.java index 5d45ebb45..673e8a4bb 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/listeners/SubsnpAccessionsStepListener.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/listeners/SubsnpAccessionsStepListener.java @@ -4,26 +4,21 @@ import org.springframework.batch.core.StepExecution; import uk.ac.ebi.eva.accession.core.batch.listeners.GenericProgressListener; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; -import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import uk.ac.ebi.eva.metrics.metric.MetricCompute; public class SubsnpAccessionsStepListener extends GenericProgressListener { private final MetricCompute metricCompute; - private final SubmittedVariantAccessioningService submittedVariantAccessioningService; - public SubsnpAccessionsStepListener(InputParameters inputParameters, MetricCompute metricCompute, - SubmittedVariantAccessioningService submittedVariantAccessioningService) { + public SubsnpAccessionsStepListener(InputParameters inputParameters, MetricCompute metricCompute) { super(inputParameters.getChunkSize()); this.metricCompute = metricCompute; - this.submittedVariantAccessioningService = submittedVariantAccessioningService; } @Override public ExitStatus afterStep(StepExecution stepExecution) { ExitStatus status = super.afterStep(stepExecution); - submittedVariantAccessioningService.shutDownAccessionGenerator(); metricCompute.saveMetricsCountsInDB(); return status; } diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/BeanNames.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/BeanNames.java index 4e924ac67..f6f92fd04 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/BeanNames.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/BeanNames.java @@ -35,4 +35,6 @@ public class BeanNames { public static final String CREATE_SUBSNP_ACCESSION_JOB = "CREATE_SUBSNP_ACCESSION_JOB"; public static final String SUBSNP_ACCESSION_STEP_LISTENER = "SUBSNP_ACCESSION_STEP_LISTENER"; + + public static final String ACCESSIONING_SHUTDOWN_STEP = "ACCESSIONING_SHUTDOWN_STEP"; } diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfiguration.java index 27fe57eef..03ea811e4 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfiguration.java @@ -26,6 +26,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.ACCESSIONING_SHUTDOWN_STEP; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.BUILD_REPORT_STEP; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.CHECK_SUBSNP_ACCESSION_STEP; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.CREATE_SUBSNP_ACCESSION_JOB; @@ -47,11 +48,16 @@ public class CreateSubsnpAccessionsJobConfiguration { @Qualifier(BUILD_REPORT_STEP) private Step buildReportStep; + @Autowired + @Qualifier(ACCESSIONING_SHUTDOWN_STEP) + private Step accessioningShutdownStep; + @Bean(CREATE_SUBSNP_ACCESSION_JOB) public Job createSubsnpAccessionJob(JobBuilderFactory jobBuilderFactory) { return jobBuilderFactory.get(CREATE_SUBSNP_ACCESSION_JOB) .incrementer(new RunIdIncrementer()) .start(createSubsnpAccessionStep) + .next(accessioningShutdownStep) .next(buildReportStep) .next(checkSubsnpAccessionStep) .build(); diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/listeners/ListenersConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/listeners/ListenersConfiguration.java index 6150bbc7c..bb183662d 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/listeners/ListenersConfiguration.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/listeners/ListenersConfiguration.java @@ -5,7 +5,6 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.web.client.RestTemplate; -import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.pipeline.batch.listeners.SubsnpAccessionsStepListener; import uk.ac.ebi.eva.accession.pipeline.metric.AccessioningMetricCompute; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; @@ -19,9 +18,8 @@ @Import({MetricConfiguration.class}) public class ListenersConfiguration { @Bean(SUBSNP_ACCESSION_STEP_LISTENER) - public SubsnpAccessionsStepListener clusteringProgressListener(InputParameters parameters, MetricCompute metricCompute, - SubmittedVariantAccessioningService submittedVariantAccessioningService) { - return new SubsnpAccessionsStepListener(parameters, metricCompute, submittedVariantAccessioningService); + public SubsnpAccessionsStepListener clusteringProgressListener(InputParameters parameters, MetricCompute metricCompute) { + return new SubsnpAccessionsStepListener(parameters, metricCompute); } @Bean diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/AccessioningShutdownStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/AccessioningShutdownStepConfiguration.java new file mode 100644 index 000000000..75f083f2d --- /dev/null +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/AccessioningShutdownStepConfiguration.java @@ -0,0 +1,28 @@ +package uk.ac.ebi.eva.accession.pipeline.configuration.batch.steps; + +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; + +import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.ACCESSIONING_SHUTDOWN_STEP; + +@Configuration +@EnableBatchProcessing +public class AccessioningShutdownStepConfiguration { + @Autowired + private SubmittedVariantAccessioningService submittedVariantAccessioningService; + + @Bean(ACCESSIONING_SHUTDOWN_STEP) + public Step accessioningShutDownStep(StepBuilderFactory stepBuilderFactory) { + return stepBuilderFactory.get(ACCESSIONING_SHUTDOWN_STEP) + .tasklet((contribution, chunkContext) -> { + submittedVariantAccessioningService.shutDownAccessionGenerator(); + return null; + }) + .build(); + } +} \ No newline at end of file diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java index de407ec96..be62853e1 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java @@ -52,6 +52,7 @@ import java.util.TreeSet; import static org.junit.Assert.assertEquals; +import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.ACCESSIONING_SHUTDOWN_STEP; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.BUILD_REPORT_STEP; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.CHECK_SUBSNP_ACCESSION_STEP; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.CREATE_SUBSNP_ACCESSION_STEP; @@ -111,9 +112,10 @@ public void executeJob() throws Exception { } private void assertStepNames(Collection stepExecutions) { - assertEquals(3, stepExecutions.size()); + assertEquals(4, stepExecutions.size()); Iterator iterator = stepExecutions.iterator(); assertEquals(CREATE_SUBSNP_ACCESSION_STEP, iterator.next().getStepName()); + assertEquals(ACCESSIONING_SHUTDOWN_STEP, iterator.next().getStepName()); assertEquals(BUILD_REPORT_STEP, iterator.next().getStepName()); assertEquals(CHECK_SUBSNP_ACCESSION_STEP, iterator.next().getStepName()); } diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java index 3ba44e6b0..3c89c94d9 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java @@ -51,6 +51,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.ACCESSIONING_SHUTDOWN_STEP; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.BUILD_REPORT_STEP; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.CHECK_SUBSNP_ACCESSION_STEP; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.CREATE_SUBSNP_ACCESSION_STEP; @@ -128,9 +129,10 @@ private static int getVariantLineNumberByPosition(File output, String position) } private void assertStepNames(Collection stepExecutions) { - assertEquals(3, stepExecutions.size()); + assertEquals(4, stepExecutions.size()); Iterator iterator = stepExecutions.iterator(); assertEquals(CREATE_SUBSNP_ACCESSION_STEP, iterator.next().getStepName()); + assertEquals(ACCESSIONING_SHUTDOWN_STEP, iterator.next().getStepName()); assertEquals(BUILD_REPORT_STEP, iterator.next().getStepName()); assertEquals(CHECK_SUBSNP_ACCESSION_STEP, iterator.next().getStepName()); } diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java index e3754aae0..85a8e07f3 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java @@ -42,13 +42,13 @@ import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities.ContiguousIdBlock; import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.repositories.ContiguousIdBlockRepository; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; +import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.FixSpringMongoDbRule; -import uk.ac.ebi.eva.accession.pipeline.test.RecoverTestAccessioningConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; @@ -57,21 +57,23 @@ import java.net.URI; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; +import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; +import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.ACCESSIONING_SHUTDOWN_STEP; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.BUILD_REPORT_STEP; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.CHECK_SUBSNP_ACCESSION_STEP; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.CREATE_SUBSNP_ACCESSION_STEP; @RunWith(SpringRunner.class) -@ContextConfiguration(classes = {RecoverTestAccessioningConfiguration.class, BatchTestConfiguration.class, - SubmittedVariantAccessioningConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-recover-state-test.properties") public class CreateSubsnpAccessionsRecoverStateTest { private static final String TEST_DB = "test-db"; @@ -134,11 +136,11 @@ public void cleanSlate() throws Exception { Files.deleteIfExists(Paths.get(inputParameters.getFasta() + ".fai")); } - /** - * Note that for this test to work, we prepare the Mongo database in {@link RecoverTestAccessioningConfiguration}. - */ @Test public void accessionJobShouldRecoverUncommittedAccessions() throws Exception { + // Fill Uncommitted Accessions - accessions that are present in MongoDB but wasn't committed in the block service + initializeMongoDbWithUncommittedAccessions(); + verifyInitialDBState(); runAccessioningJob(); @@ -149,12 +151,10 @@ public void accessionJobShouldRecoverUncommittedAccessions() throws Exception { assertCountsInMongo(EXPECTED_VARIANTS_ACCESSIONED_FROM_VCF + 85); } - /** - * We initialize DB for the test by inserting sves in {@link RecoverTestAccessioningConfiguration}. - */ private void verifyInitialDBState() { // Contiguous Id Block DB: - // Initial state of DB is 4 blocks are "reserved" but not "committed" in postgresql + // Initial state of DB is 4 blocks are present but not "committed" in postgresql + // Initialized using "resources/test-data/contiguous_id_blocks_recover_state_data.sql" // block id first value last value last committed // 1 5000000000 5000000029 4999999999 // 2 5000000030 5000000059 5000000029 @@ -170,41 +170,34 @@ private void verifyInitialDBState() { assertEquals(85, mongoRepository.count()); // 30 + 25 + 30 assertEquals(4, blockRepository.count()); - //recover state run in constructor of the MonotonicAccessionGenerator, - // so will run when objects are being created even before job starts and will try to recover the blocks - // The below stage of DB is after the recover state has already ran - - // 1st block is recovered and last committed updated to 5000000029 as all are present in mongo + // Since none of the 4 blocks got committed - everyone's last committed value is first_value - 1 ContiguousIdBlock block1 = blockRepository.findById(1l).get(); assertEquals(5000000000l, block1.getFirstValue()); - assertEquals(5000000029l, block1.getLastCommitted()); + assertEquals(4999999999l, block1.getLastCommitted()); assertEquals(5000000029l, block1.getLastValue()); + assertEquals("test-instance-recover-state-00", block1.getApplicationInstanceId()); assertTrue(block1.isNotReserved()); - // 2nd block's committed is updated 5000000034 as there are available accessions after that ContiguousIdBlock block2 = blockRepository.findById(2l).get(); assertEquals(5000000030l, block2.getFirstValue()); - assertEquals(5000000034l, block2.getLastCommitted()); + assertEquals(5000000029l, block2.getLastCommitted()); assertEquals(5000000059l, block2.getLastValue()); - assertTrue(block2.isReserved()); - - // 3rd block is not updated even though it is full (all accessions of this block are present in mongo) - // the current algorithm takes the uncompleted blocks in ascending order of last value - // and stops updating blocks as soon as it finds a block which is not full - // In our case 1st is picked and updated, - // 2nd is picked and updated but is not full - // the algorithm stops at this point and did not bother to check the 3rd block + assertEquals("test-instance-recover-state-00", block1.getApplicationInstanceId()); + assertTrue(block2.isNotReserved()); + ContiguousIdBlock block3 = blockRepository.findById(3l).get(); assertEquals(5000000060l, block3.getFirstValue()); assertEquals(5000000059l, block3.getLastCommitted()); assertEquals(5000000089l, block3.getLastValue()); - assertTrue(block3.isReserved()); + assertEquals("test-instance-recover-state-00", block1.getApplicationInstanceId()); + assertTrue(block3.isNotReserved()); ContiguousIdBlock block4 = blockRepository.findById(4l).get(); assertEquals(5000000090l, block4.getFirstValue()); assertEquals(5000000089l, block4.getLastCommitted()); assertEquals(5000000119l, block4.getLastValue()); - assertTrue(block4.isReserved()); + assertEquals("test-instance-recover-state-00", block1.getApplicationInstanceId()); + assertTrue(block4.isNotReserved()); } private void verifyEndDBState() { @@ -213,24 +206,32 @@ private void verifyEndDBState() { assertEquals(107, mongoRepository.count()); // 85 (already present) + 22 (accessioned) assertEquals(4, blockRepository.count()); + /* + * Accessions that were already present in mongo but not updated in block's last committed were recovered. + * */ + + // Block Recovered - (No accession used from this block as entire block was already used) ContiguousIdBlock block1 = blockRepository.findById(1l).get(); assertEquals(5000000000l, block1.getFirstValue()); assertEquals(5000000029l, block1.getLastCommitted()); assertEquals(5000000029l, block1.getLastValue()); + assertEquals("test-instance-recover-state-01", block1.getApplicationInstanceId()); assertTrue(block1.isNotReserved()); - // used the 5 unused accessions 5000000030 to 5000000034 + // Block Recovered - (used the 5 unused accessions 5000000035 to 5000000039 and recovered others) ContiguousIdBlock block2 = blockRepository.findById(2l).get(); assertEquals(5000000030l, block2.getFirstValue()); assertEquals(5000000059l, block2.getLastCommitted()); assertEquals(5000000059l, block2.getLastValue()); + assertEquals("test-instance-recover-state-01", block1.getApplicationInstanceId()); assertTrue(block2.isNotReserved()); - // Now that the 2nd block is full and committed, 3rd blocks also get's picked up and its last committed updated + // Block Recovered - (No accession used from this block as entire block was already used) ContiguousIdBlock block3 = blockRepository.findById(3l).get(); assertEquals(5000000060l, block3.getFirstValue()); assertEquals(5000000089l, block3.getLastCommitted()); assertEquals(5000000089l, block3.getLastValue()); + assertEquals("test-instance-recover-state-01", block1.getApplicationInstanceId()); assertTrue(block3.isNotReserved()); // used the remaining 17 (22 - 5 (2nd block)) from 4th block @@ -238,6 +239,7 @@ private void verifyEndDBState() { assertEquals(5000000090l, block4.getFirstValue()); assertEquals(5000000106l, block4.getLastCommitted()); assertEquals(5000000119l, block4.getLastValue()); + assertEquals("test-instance-recover-state-01", block1.getApplicationInstanceId()); assertTrue(block4.isNotReserved()); } @@ -248,9 +250,10 @@ private void runAccessioningJob() throws Exception { } private void assertStepNames(Collection stepExecutions) { - assertEquals(3, stepExecutions.size()); + assertEquals(4, stepExecutions.size()); Iterator iterator = stepExecutions.iterator(); assertEquals(CREATE_SUBSNP_ACCESSION_STEP, iterator.next().getStepName()); + assertEquals(ACCESSIONING_SHUTDOWN_STEP, iterator.next().getStepName()); assertEquals(BUILD_REPORT_STEP, iterator.next().getStepName()); assertEquals(CHECK_SUBSNP_ACCESSION_STEP, iterator.next().getStepName()); } @@ -265,4 +268,49 @@ private void assertCountsInVcfReport(int expected) throws IOException { assertEquals(expected, numVariantsInReport); } + private void initializeMongoDbWithUncommittedAccessions() { + mongoRepository.deleteAll(); + + List submittedVariantEntityList = new ArrayList<>(); + // Entries for 1st block + for(long i=5000000000l;i<=5000000029l;i++){ + SubmittedVariant model = new SubmittedVariant("assembly", 1111, + "project", "contig", 100, "A", "T", + null, false, false, false, + false, null); + SubmittedVariantEntity entity = new SubmittedVariantEntity(i, "hash"+i, model, 1); + submittedVariantEntityList.add(entity); + } + + // Entries for 2nd block + for(long i=5000000030l;i<=5000000034l;i++){ + SubmittedVariant model = new SubmittedVariant("assembly", 1111, + "project", "contig", 100, "A", "T", + null, false, false, false, + false, null); + SubmittedVariantEntity entity = new SubmittedVariantEntity(i, "hash"+i, model, 1); + submittedVariantEntityList.add(entity); + } + for(long i=5000000040l;i<=5000000059l;i++){ + SubmittedVariant model = new SubmittedVariant("assembly", 1111, + "project", "contig", 100, "A", "T", + null, false, false, false, + false, null); + SubmittedVariantEntity entity = new SubmittedVariantEntity(i, "hash"+i, model, 1); + submittedVariantEntityList.add(entity); + } + + // Entries for 3rd block + for(long i=5000000060l;i<=5000000089l;i++){ + SubmittedVariant model = new SubmittedVariant("assembly", 1111, + "project", "contig", 100, "A", "T", + null, false, false, false, + false, null); + SubmittedVariantEntity entity = new SubmittedVariantEntity(i, "hash"+i, model, 1); + submittedVariantEntityList.add(entity); + } + + mongoRepository.saveAll(submittedVariantEntityList); + } + } diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoveringStateJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoveringStateJobConfigurationTest.java index 434fd2f6d..d1aae1e7d 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoveringStateJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoveringStateJobConfigurationTest.java @@ -33,7 +33,6 @@ import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; -import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; @@ -44,13 +43,13 @@ import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.repositories.ContiguousIdBlockRepository; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; +import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionReportWriter; import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; import uk.ac.ebi.eva.accession.pipeline.test.FixSpringMongoDbRule; -import uk.ac.ebi.eva.accession.pipeline.test.RecoveringAccessioningConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; @@ -66,13 +65,13 @@ import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; +import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.ACCESSIONING_SHUTDOWN_STEP; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.BUILD_REPORT_STEP; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.CHECK_SUBSNP_ACCESSION_STEP; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.CREATE_SUBSNP_ACCESSION_STEP; @RunWith(SpringRunner.class) -@ContextConfiguration(classes = {RecoveringAccessioningConfiguration.class, BatchTestConfiguration.class, - SubmittedVariantAccessioningConfiguration.class}) +@ContextConfiguration(classes = {BatchTestConfiguration.class, SubmittedVariantAccessioningConfiguration.class}) @TestPropertySource("classpath:accession-pipeline-recover-test.properties") public class CreateSubsnpAccessionsRecoveringStateJobConfigurationTest { @@ -138,9 +137,6 @@ public void cleanSlate() throws Exception { Files.deleteIfExists(Paths.get(inputParameters.getFasta() + ".fai")); } - /** - * Note that for this test to work, we prepare the Mongo database in {@link RecoveringAccessioningConfiguration}. - */ @Test public void accessionJobShouldRecoverUncommittedAccessions() throws Exception { startWithAnAccessionInMongoNotCommittedInTheBlockService(); @@ -156,12 +152,18 @@ public void accessionJobShouldRecoverUncommittedAccessions() throws Exception { } private void startWithAnAccessionInMongoNotCommittedInTheBlockService() { + repository.deleteAll(); + SubmittedVariant model = new SubmittedVariant("assembly", 1111, "project", "contig", 100, "A", "T", null, false, + false, false, false, null); + SubmittedVariantEntity entity = new SubmittedVariantEntity(UNCOMMITTED_ACCESSION, "hash-10", model, 1); + repository.save(entity); + assertEquals(1, repository.count()); assertEquals(1, repository.findByAccession(UNCOMMITTED_ACCESSION).size()); assertEquals(1, blockRepository.count()); // This means that the last committed accession is the previous one to the UNCOMMITTED_ACCESSION - assertEquals(UNCOMMITTED_ACCESSION, blockRepository.findAll().iterator().next().getLastCommitted()); + assertEquals(UNCOMMITTED_ACCESSION-1, blockRepository.findAll().iterator().next().getLastCommitted()); } private void runJob() throws Exception { @@ -171,9 +173,10 @@ private void runJob() throws Exception { } private void assertStepNames(Collection stepExecutions) { - assertEquals(3, stepExecutions.size()); + assertEquals(4, stepExecutions.size()); Iterator iterator = stepExecutions.iterator(); assertEquals(CREATE_SUBSNP_ACCESSION_STEP, iterator.next().getStepName()); + assertEquals(ACCESSIONING_SHUTDOWN_STEP, iterator.next().getStepName()); assertEquals(BUILD_REPORT_STEP, iterator.next().getStepName()); assertEquals(CHECK_SUBSNP_ACCESSION_STEP, iterator.next().getStepName()); } diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchTestConfiguration.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchTestConfiguration.java index 1265348ae..571932e82 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchTestConfiguration.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/BatchTestConfiguration.java @@ -32,6 +32,7 @@ import uk.ac.ebi.eva.accession.pipeline.configuration.batch.processors.VariantProcessorConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.io.VcfReaderConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.jobs.CreateSubsnpAccessionsJobConfiguration; +import uk.ac.ebi.eva.accession.pipeline.configuration.batch.steps.AccessioningShutdownStepConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.steps.BuildReportStepConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.steps.CheckSubsnpAccessionsStepConfiguration; import uk.ac.ebi.eva.accession.pipeline.configuration.batch.steps.CreateSubsnpAccessionsStepConfiguration; @@ -46,7 +47,7 @@ @Import({CreateSubsnpAccessionsJobConfiguration.class, CreateSubsnpAccessionsStepConfiguration.class, CheckSubsnpAccessionsStepConfiguration.class, VcfReaderConfiguration.class, VariantProcessorConfiguration.class, AccessionWriterConfiguration.class, - BuildReportStepConfiguration.class, + BuildReportStepConfiguration.class, AccessioningShutdownStepConfiguration.class, ChunkSizeCompletionPolicyConfiguration.class, InvalidVariantSkipPolicyConfiguration.class, EvaAccessionJobLauncherCommandLineRunner.class}) public class BatchTestConfiguration { diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/RecoverTestAccessioningConfiguration.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/RecoverTestAccessioningConfiguration.java deleted file mode 100644 index 7cd7c3582..000000000 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/RecoverTestAccessioningConfiguration.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright 2019 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package uk.ac.ebi.eva.accession.pipeline.test; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicAccessionGenerator; -import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; -import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.ContiguousIdBlockService; - -import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; -import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; -import uk.ac.ebi.eva.accession.core.configuration.ApplicationProperties; -import uk.ac.ebi.eva.accession.core.service.nonhuman.eva.SubmittedVariantAccessioningDatabaseService; -import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; -import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; -import uk.ac.ebi.eva.accession.core.service.nonhuman.eva.SubmittedVariantMonotonicAccessioningService; -import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; - -import java.util.ArrayList; -import java.util.List; - -/** - * This configuration class has the single purpose of having loaded in MongoDB an object *before* the - * MonotonicAccessionGenerator is instantiated (and autowired in the accessioning service and pipeline jobs) so that - * the generator can recover from uncommitted accessions. - * - * An uncommitted accession is an accession that is present in MongoDB but wasn't committed in the block service (e.g. - * due to an unexpected crash of the application in previous executions). If the block service doesn't recover, this - * might lead to a single accession being assigned to several different objects in mongo. - */ -@Configuration -public class RecoverTestAccessioningConfiguration { - - private static final Logger logger = LoggerFactory.getLogger(RecoveringAccessioningConfiguration.class); - - @Bean - public SubmittedVariantMonotonicAccessioningService submittedVariantMonotonicAccessioningService( - @Autowired @Qualifier("testSubmittedVariantAccessionGeneratorRecover") - MonotonicAccessionGenerator accessionGenerator, - @Autowired SubmittedVariantAccessioningDatabaseService databaseService) { - SubmittedVariantMonotonicAccessioningService sveMonotonicAccService = new SubmittedVariantMonotonicAccessioningService(accessionGenerator, - databaseService, - new SubmittedVariantSummaryFunction(), - new SHA1HashingFunction()); - sveMonotonicAccService.shutDownAccessioning(); - return sveMonotonicAccService; - } - - @Bean("testSubmittedVariantAccessionGeneratorRecover") - public MonotonicAccessionGenerator testSubmittedVariantAccessionGeneratorRecover( - @Autowired SubmittedVariantAccessioningRepository repository, - @Autowired SubmittedVariantAccessioningDatabaseService databaseService, - @Autowired ApplicationProperties properties, - @Autowired ContiguousIdBlockService blockService) { - - repository.deleteAll(); - - List submittedVariantEntityList = new ArrayList<>(); - // Entries for 1st block - for(long i=5000000000l;i<=5000000029l;i++){ - SubmittedVariant model = new SubmittedVariant("assembly", 1111, - "project", "contig", 100, "A", "T", - null, false, false, false, - false, null); - SubmittedVariantEntity entity = new SubmittedVariantEntity(i, "hash"+i, model, 1); - submittedVariantEntityList.add(entity); - } - - // Entries for 2nd block - for(long i=5000000030l;i<=5000000034l;i++){ - SubmittedVariant model = new SubmittedVariant("assembly", 1111, - "project", "contig", 100, "A", "T", - null, false, false, false, - false, null); - SubmittedVariantEntity entity = new SubmittedVariantEntity(i, "hash"+i, model, 1); - submittedVariantEntityList.add(entity); - } - for(long i=5000000040l;i<=5000000059l;i++){ - SubmittedVariant model = new SubmittedVariant("assembly", 1111, - "project", "contig", 100, "A", "T", - null, false, false, false, - false, null); - SubmittedVariantEntity entity = new SubmittedVariantEntity(i, "hash"+i, model, 1); - submittedVariantEntityList.add(entity); - } - - // Entries for 3rd block - for(long i=5000000060l;i<=5000000089l;i++){ - SubmittedVariant model = new SubmittedVariant("assembly", 1111, - "project", "contig", 100, "A", "T", - null, false, false, false, - false, null); - SubmittedVariantEntity entity = new SubmittedVariantEntity(i, "hash"+i, model, 1); - submittedVariantEntityList.add(entity); - } - - repository.saveAll(submittedVariantEntityList); - - return new MonotonicAccessionGenerator<>(properties.getSubmitted().getCategoryId(), - properties.getInstanceId(), - blockService, - databaseService); - } -} diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/RecoveringAccessioningConfiguration.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/RecoveringAccessioningConfiguration.java deleted file mode 100644 index 011162b81..000000000 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/RecoveringAccessioningConfiguration.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright 2019 EMBL - European Bioinformatics Institute - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package uk.ac.ebi.eva.accession.pipeline.test; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicAccessionGenerator; -import uk.ac.ebi.ampt2d.commons.accession.hashing.SHA1HashingFunction; -import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.ContiguousIdBlockService; - -import uk.ac.ebi.eva.accession.core.model.ISubmittedVariant; -import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; -import uk.ac.ebi.eva.accession.core.configuration.ApplicationProperties; -import uk.ac.ebi.eva.accession.core.service.nonhuman.eva.SubmittedVariantAccessioningDatabaseService; -import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; -import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; -import uk.ac.ebi.eva.accession.core.service.nonhuman.eva.SubmittedVariantMonotonicAccessioningService; -import uk.ac.ebi.eva.accession.core.summary.SubmittedVariantSummaryFunction; - -import static uk.ac.ebi.eva.accession.pipeline.configuration.batch.jobs.CreateSubsnpAccessionsRecoveringStateJobConfigurationTest.UNCOMMITTED_ACCESSION; - -/** - * This configuration class has the single purpose of having loaded in MongoDB an object *before* the - * MonotonicAccessionGenerator is instantiated (and autowired in the accessioning service and pipeline jobs) so that - * the generator can recover from uncommitted accessions. - * - * An uncommitted accession is an accession that is present in MongoDB but wasn't committed in the block service (e.g. - * due to an unexpected crash of the application in previous executions). If the block service doesn't recover, this - * might lead to a single accession being assigned to several different objects in mongo. - */ -@Configuration -public class RecoveringAccessioningConfiguration { - - private static final Logger logger = LoggerFactory.getLogger(RecoveringAccessioningConfiguration.class); - - @Bean - public SubmittedVariantMonotonicAccessioningService submittedVariantMonotonicAccessioningService( - @Autowired @Qualifier("testSubmittedVariantAccessionGenerator") - MonotonicAccessionGenerator accessionGenerator, - @Autowired SubmittedVariantAccessioningDatabaseService databaseService) { - SubmittedVariantMonotonicAccessioningService submittedVariantMonotonicAccessioningService = new SubmittedVariantMonotonicAccessioningService(accessionGenerator, - databaseService, - new SubmittedVariantSummaryFunction(), - new SHA1HashingFunction()); - submittedVariantMonotonicAccessioningService.shutDownAccessioning(); - return submittedVariantMonotonicAccessioningService; - } - - @Bean("testSubmittedVariantAccessionGenerator") - public MonotonicAccessionGenerator testSubmittedVariantAccessionGenerator( - @Autowired SubmittedVariantAccessioningRepository repository, - @Autowired SubmittedVariantAccessioningDatabaseService databaseService, - @Autowired ApplicationProperties properties, - @Autowired ContiguousIdBlockService blockService) { - SubmittedVariant model = new SubmittedVariant("assembly", 1111, "project", "contig", 100, "A", "T", null, false, - false, false, false, null); - - SubmittedVariantEntity entity = new SubmittedVariantEntity(UNCOMMITTED_ACCESSION, "hash-10", model, 1); - - repository.deleteAll(); - repository.save(entity); - logger.warn( - "Saved an entity without committing its accession {} in the block service. This should only appear in" - + " tests.", - UNCOMMITTED_ACCESSION); - - return new MonotonicAccessionGenerator<>(properties.getSubmitted().getCategoryId(), - properties.getInstanceId(), - blockService, - databaseService); - } -} diff --git a/eva-accession-pipeline/src/test/resources/test-data/contiguous_id_blocks_recover_state_data.sql b/eva-accession-pipeline/src/test/resources/test-data/contiguous_id_blocks_recover_state_data.sql index 998ae8a5d..0e6c759d5 100644 --- a/eva-accession-pipeline/src/test/resources/test-data/contiguous_id_blocks_recover_state_data.sql +++ b/eva-accession-pipeline/src/test/resources/test-data/contiguous_id_blocks_recover_state_data.sql @@ -1,4 +1,4 @@ -INSERT INTO contiguous_id_blocks VALUES(1, 'test-instance-recover-state-01', 'test-pipeline-recover-state-ss', 5000000000, 4999999999, 5000000029, false, '1970-01-01 00:00:00'); -INSERT INTO contiguous_id_blocks VALUES(2, 'test-instance-recover-state-01', 'test-pipeline-recover-state-ss', 5000000030, 5000000029, 5000000059, false, '1970-01-01 00:00:00'); -INSERT INTO contiguous_id_blocks VALUES(3, 'test-instance-recover-state-01', 'test-pipeline-recover-state-ss', 5000000060, 5000000059, 5000000089, false, '1970-01-01 00:00:00'); -INSERT INTO contiguous_id_blocks VALUES(4, 'test-instance-recover-state-01', 'test-pipeline-recover-state-ss', 5000000090, 5000000089, 5000000119, false, '1970-01-01 00:00:00'); \ No newline at end of file +INSERT INTO contiguous_id_blocks VALUES(1, 'test-instance-recover-state-00', 'test-pipeline-recover-state-ss', 5000000000, 4999999999, 5000000029, false, '1970-01-01 00:00:00'); +INSERT INTO contiguous_id_blocks VALUES(2, 'test-instance-recover-state-00', 'test-pipeline-recover-state-ss', 5000000030, 5000000029, 5000000059, false, '1970-01-01 00:00:00'); +INSERT INTO contiguous_id_blocks VALUES(3, 'test-instance-recover-state-00', 'test-pipeline-recover-state-ss', 5000000060, 5000000059, 5000000089, false, '1970-01-01 00:00:00'); +INSERT INTO contiguous_id_blocks VALUES(4, 'test-instance-recover-state-00', 'test-pipeline-recover-state-ss', 5000000090, 5000000089, 5000000119, false, '1970-01-01 00:00:00'); \ No newline at end of file diff --git a/pom.xml b/pom.xml index 583194093..c905e045c 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ UTF-8 4.13 0.8.5 - 0.7.13-SNAPSHOT + 0.7.13