diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/recovery/RSAccessionRecoveryService.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/recovery/RSAccessionRecoveryService.java new file mode 100644 index 000000000..5f62f129a --- /dev/null +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/recovery/RSAccessionRecoveryService.java @@ -0,0 +1,28 @@ +package uk.ac.ebi.eva.accession.clustering.batch.recovery; + +import org.springframework.batch.core.JobExecution; +import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicAccessionRecoveryAgent; + +import java.time.LocalDateTime; + +public class RSAccessionRecoveryService { + private final static String CATEGORY_ID = "rs"; + private MonotonicAccessionRecoveryAgent monotonicAccessionRecoveryAgent; + private JobExecution jobExecution; + private long recoveryCutOffDays; + + public RSAccessionRecoveryService(MonotonicAccessionRecoveryAgent monotonicAccessionRecoveryAgent, + long recoveryCutOffDays) { + this.monotonicAccessionRecoveryAgent = monotonicAccessionRecoveryAgent; + this.recoveryCutOffDays = recoveryCutOffDays; + } + + public void runRecoveryForCategoryRS() { + LocalDateTime recoveryCutOffTime = LocalDateTime.now().minusDays(recoveryCutOffDays); + monotonicAccessionRecoveryAgent.runRecovery(CATEGORY_ID, jobExecution.getJobId().toString(), recoveryCutOffTime); + } + + public void setJobExecution(JobExecution jobExecution) { + this.jobExecution = jobExecution; + } +} diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/BeanNames.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/BeanNames.java index f5dbd5cfe..06a49ab68 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/BeanNames.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/BeanNames.java @@ -55,6 +55,8 @@ public class BeanNames { public static final String PROGRESS_LISTENER = "PROGRESS_LISTENER"; + public static final String JOB_EXECUTION_LISTENER = "JOB_EXECUTION_LISTENER"; + public static final String ACCESSIONING_SHUTDOWN_STEP = "ACCESSIONING_SHUTDOWN_STEP"; public static final String CLUSTERING_FROM_VCF_STEP = "CLUSTERING_FROM_VCF_STEP"; @@ -95,4 +97,12 @@ public class BeanNames { public static final String CLUSTERED_CLUSTERING_WRITER_JOB_EXECUTION_SETTER = "CLUSTERED_CLUSTERING_WRITER_JOB_EXECUTION_SETTER"; public static final String RS_SPLIT_WRITER_JOB_EXECUTION_SETTER = "RS_SPLIT_WRITER_JOB_EXECUTION_SETTER"; + + public static final String RS_ACCESSION_RECOVERY_SERVICE = "RS_ACCESSION_RECOVERY_SERVICE"; + + public static final String RS_ACCESSION_RECOVERY_STEP = "RS_ACCESSION_RECOVERY_STEP"; + + public static final String RS_ACCESSION_RECOVERY_JOB = "RS_ACCESSION_RECOVERY_JOB"; + + public static final String RS_ACCESSION_RECOVERY_JOB_LISTENER = "RS_ACCESSION_RECOVERY_JOB_LISTENER"; } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java index b9517c10a..2399d090c 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusterUnclusteredVariantsJobConfiguration.java @@ -16,6 +16,7 @@ package uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs; import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; @@ -28,6 +29,7 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTER_UNCLUSTERED_VARIANTS_JOB; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.JOB_EXECUTION_LISTENER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_MERGE_CANDIDATES_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_SPLIT_CANDIDATES_STEP; @@ -44,6 +46,7 @@ public Job clusteringFromMongoJob( @Qualifier(CLEAR_RS_MERGE_AND_SPLIT_CANDIDATES_STEP) Step clearRSMergeAndSplitCandidatesStep, @Qualifier(CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP) Step clusteringNonClusteredVariantsFromMongoStep, @Qualifier(ACCESSIONING_SHUTDOWN_STEP) Step accessioningShutdownStep, + @Qualifier(JOB_EXECUTION_LISTENER) JobExecutionListener jobExecutionListener, JobBuilderFactory jobBuilderFactory) { return jobBuilderFactory.get(CLUSTER_UNCLUSTERED_VARIANTS_JOB) .incrementer(new RunIdIncrementer()) @@ -52,6 +55,7 @@ public Job clusteringFromMongoJob( .next(clearRSMergeAndSplitCandidatesStep) .next(clusteringNonClusteredVariantsFromMongoStep) .next(accessioningShutdownStep) + .listener(jobExecutionListener) .build(); } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java index 786900cd2..ea1c4a653 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringFromMongoJobConfiguration.java @@ -18,6 +18,7 @@ import htsjdk.samtools.util.StringUtil; import org.springframework.batch.core.Job; import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; import org.springframework.batch.core.StepContribution; import org.springframework.batch.core.StepExecution; @@ -47,6 +48,7 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_CLUSTERED_VARIANTS_FROM_MONGO_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_FROM_MONGO_JOB; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.JOB_EXECUTION_LISTENER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_MERGE_CANDIDATES_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_SPLIT_CANDIDATES_STEP; @@ -86,6 +88,7 @@ public Job clusteringFromMongoJob(@Qualifier(CLUSTERING_CLUSTERED_VARIANTS_FROM_ // Back-propagate RS in the remapped assembly that were split or merged @Qualifier(BACK_PROPAGATE_SPLIT_OR_MERGED_RS_STEP) Step backPropagateSplitMergedRSStep, + @Qualifier(JOB_EXECUTION_LISTENER) JobExecutionListener jobExecutionListener, StepBuilderFactory stepBuilderFactory, JobBuilderFactory jobBuilderFactory, InputParameters inputParameters) { @@ -95,6 +98,7 @@ public Job clusteringFromMongoJob(@Qualifier(CLUSTERING_CLUSTERED_VARIANTS_FROM_ .incrementer(new RunIdIncrementer()) //We need the dummy step here because Spring won't conditionally start the first step .start(dummyStep) + .listener(jobExecutionListener) .next(jobExecutionDecider) .on("TRUE") .to(new FlowBuilder("remappedAssemblyClusteringFlow") @@ -105,7 +109,8 @@ public Job clusteringFromMongoJob(@Qualifier(CLUSTERING_CLUSTERED_VARIANTS_FROM_ .next(clusteringNonClusteredVariantsFromMongoStep) .next(accessioningShutdownStep) .next(backPropagateNewRSStep) - .next(backPropagateSplitMergedRSStep).build()) + .next(backPropagateSplitMergedRSStep) + .build()) .on("*").end() .from(jobExecutionDecider) .on("FALSE") diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryJobConfiguration.java new file mode 100644 index 000000000..d9d80cafc --- /dev/null +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryJobConfiguration.java @@ -0,0 +1,39 @@ +package uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs; + +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobExecutionListener; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_ACCESSION_RECOVERY_JOB; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_ACCESSION_RECOVERY_JOB_LISTENER; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_ACCESSION_RECOVERY_STEP; + +@Configuration +@EnableBatchProcessing +public class RSAccessionRecoveryJobConfiguration { + + @Autowired + @Qualifier(RS_ACCESSION_RECOVERY_STEP) + private Step monotonicAccessionRecoveryAgentCategoryRSStep; + + @Autowired + @Qualifier(RS_ACCESSION_RECOVERY_JOB_LISTENER) + private JobExecutionListener monotonicAccessionRecoveryAgentCategoryRSJobListener; + + @Bean(RS_ACCESSION_RECOVERY_JOB) + public Job createMonotonicAccessionRecoveryAgentCategoryRSJob(JobBuilderFactory jobBuilderFactory) { + return jobBuilderFactory.get(RS_ACCESSION_RECOVERY_JOB) + .incrementer(new RunIdIncrementer()) + .start(monotonicAccessionRecoveryAgentCategoryRSStep) + .listener(monotonicAccessionRecoveryAgentCategoryRSJobListener) + .build(); + } + +} diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java index 4b4262503..152e04ceb 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/StudyClusteringJobConfiguration.java @@ -16,6 +16,7 @@ package uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs; import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; @@ -25,6 +26,7 @@ import org.springframework.context.annotation.Configuration; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.ACCESSIONING_SHUTDOWN_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.JOB_EXECUTION_LISTENER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.STUDY_CLUSTERING_JOB; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.STUDY_CLUSTERING_STEP; @@ -35,11 +37,13 @@ public class StudyClusteringJobConfiguration { @Bean(STUDY_CLUSTERING_JOB) public Job studyClusteringJob(@Qualifier(STUDY_CLUSTERING_STEP) Step clusteringStep, @Qualifier(ACCESSIONING_SHUTDOWN_STEP) Step accessioningShutdownStep, + @Qualifier(JOB_EXECUTION_LISTENER) JobExecutionListener jobExecutionListener, JobBuilderFactory jobBuilderFactory) { return jobBuilderFactory.get(STUDY_CLUSTERING_JOB) .incrementer(new RunIdIncrementer()) .start(clusteringStep) .next(accessioningShutdownStep) + .listener(jobExecutionListener) .build(); } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/listeners/JobExecutionSetter.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/listeners/JobExecutionSetterConfiguration.java similarity index 96% rename from eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/listeners/JobExecutionSetter.java rename to eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/listeners/JobExecutionSetterConfiguration.java index 019b68b34..1bf6d0c49 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/batch/listeners/JobExecutionSetter.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/listeners/JobExecutionSetterConfiguration.java @@ -1,4 +1,4 @@ -package uk.ac.ebi.eva.accession.clustering.batch.listeners; +package uk.ac.ebi.eva.accession.clustering.configuration.batch.listeners; import org.springframework.batch.core.ExitStatus; import org.springframework.batch.core.StepExecution; @@ -17,7 +17,7 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_SPLIT_WRITER_JOB_EXECUTION_SETTER; @Configuration -public class JobExecutionSetter { +public class JobExecutionSetterConfiguration { @Bean(NON_CLUSTERED_CLUSTERING_WRITER_JOB_EXECUTION_SETTER) public StepExecutionListener getNonClusteredClusteringWriterJobExecutionSetter( @Qualifier(NON_CLUSTERED_CLUSTERING_WRITER) ClusteringWriter nonClusteredClusteringWriter) { diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/listeners/ListenersConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/listeners/ListenersConfiguration.java index e0bed8a5b..64507ea7c 100644 --- a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/listeners/ListenersConfiguration.java +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/listeners/ListenersConfiguration.java @@ -1,5 +1,7 @@ package uk.ac.ebi.eva.accession.clustering.configuration.batch.listeners; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobExecutionListener; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -8,16 +10,18 @@ import uk.ac.ebi.eva.accession.clustering.batch.listeners.ClusteringProgressListener; import uk.ac.ebi.eva.accession.clustering.metric.ClusteringMetricCompute; import uk.ac.ebi.eva.accession.clustering.parameters.InputParameters; +import uk.ac.ebi.eva.accession.core.service.nonhuman.ClusteredVariantAccessioningService; +import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.metrics.configuration.MetricConfiguration; import uk.ac.ebi.eva.metrics.count.CountServiceParameters; import uk.ac.ebi.eva.metrics.metric.MetricCompute; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.JOB_EXECUTION_LISTENER; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROGRESS_LISTENER; @Configuration @Import({MetricConfiguration.class}) public class ListenersConfiguration { - @Bean(PROGRESS_LISTENER) public ClusteringProgressListener clusteringProgressListener(InputParameters parameters, MetricCompute metricCompute) { return new ClusteringProgressListener(parameters, metricCompute); @@ -28,6 +32,21 @@ public MetricCompute getClusteringMetricCompute(CountServiceParameters countServ @Qualifier("COUNT_STATS_REST_TEMPLATE") RestTemplate restTemplate, InputParameters inputParameters) { return new ClusteringMetricCompute(countServiceParameters, restTemplate, inputParameters.getAssemblyAccession(), - inputParameters.getProjects()); + inputParameters.getProjects()); + } + + @Bean(JOB_EXECUTION_LISTENER) + public JobExecutionListener jobExecutionListener(SubmittedVariantAccessioningService submittedVariantAccessioningService, + ClusteredVariantAccessioningService clusteredVariantAccessioningService) { + return new JobExecutionListener() { + @Override + public void beforeJob(JobExecution jobExecution) {} + + @Override + public void afterJob(JobExecution jobExecution) { + submittedVariantAccessioningService.shutDownAccessionGenerator(); + clusteredVariantAccessioningService.shutDownAccessionGenerator(); + } + }; } } diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/listeners/RSAccessionRecoveryJobListenerConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/listeners/RSAccessionRecoveryJobListenerConfiguration.java new file mode 100644 index 000000000..b6e1016fa --- /dev/null +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/listeners/RSAccessionRecoveryJobListenerConfiguration.java @@ -0,0 +1,29 @@ +package uk.ac.ebi.eva.accession.clustering.configuration.batch.listeners; + +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobExecutionListener; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import uk.ac.ebi.eva.accession.clustering.batch.recovery.RSAccessionRecoveryService; + +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_ACCESSION_RECOVERY_JOB_LISTENER; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_ACCESSION_RECOVERY_SERVICE; + +@Configuration +public class RSAccessionRecoveryJobListenerConfiguration { + @Bean(RS_ACCESSION_RECOVERY_JOB_LISTENER) + public JobExecutionListener jobExecutionListener(@Qualifier(RS_ACCESSION_RECOVERY_SERVICE) + RSAccessionRecoveryService RSAccessionRecoveryService) { + return new JobExecutionListener() { + @Override + public void beforeJob(JobExecution jobExecution) { + RSAccessionRecoveryService.setJobExecution(jobExecution); + } + + @Override + public void afterJob(JobExecution jobExecution) { + } + }; + } +} diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/recovery/RSAccessionRecoveryServiceConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/recovery/RSAccessionRecoveryServiceConfiguration.java new file mode 100644 index 000000000..55c71b48a --- /dev/null +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/recovery/RSAccessionRecoveryServiceConfiguration.java @@ -0,0 +1,32 @@ +package uk.ac.ebi.eva.accession.clustering.configuration.batch.recovery; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicAccessionRecoveryAgent; +import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.ContiguousIdBlockService; +import uk.ac.ebi.eva.accession.clustering.batch.recovery.RSAccessionRecoveryService; +import uk.ac.ebi.eva.accession.core.service.nonhuman.eva.ClusteredVariantAccessioningDatabaseService; + +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_ACCESSION_RECOVERY_SERVICE; + +@Configuration +public class RSAccessionRecoveryServiceConfiguration { + @Autowired + private ContiguousIdBlockService blockService; + @Autowired + private ClusteredVariantAccessioningDatabaseService clusteredVariantAccessioningDatabaseService; + + @Value("${recovery.cutoff.days}") + private long recoveryCutOffDays; + + @Bean(RS_ACCESSION_RECOVERY_SERVICE) + public RSAccessionRecoveryService getMonotonicAccessionRecoveryAgentCategoryRSService() { + return new RSAccessionRecoveryService(getMonotonicAccessionRecoveryAgent(), recoveryCutOffDays); + } + + private MonotonicAccessionRecoveryAgent getMonotonicAccessionRecoveryAgent() { + return new MonotonicAccessionRecoveryAgent(blockService, clusteredVariantAccessioningDatabaseService); + } +} diff --git a/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/RSAccessionRecoveryStepConfiguration.java b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/RSAccessionRecoveryStepConfiguration.java new file mode 100644 index 000000000..a0514439d --- /dev/null +++ b/eva-accession-clustering/src/main/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/steps/RSAccessionRecoveryStepConfiguration.java @@ -0,0 +1,31 @@ +package uk.ac.ebi.eva.accession.clustering.configuration.batch.steps; + +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import uk.ac.ebi.eva.accession.clustering.batch.recovery.RSAccessionRecoveryService; + +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_ACCESSION_RECOVERY_SERVICE; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_ACCESSION_RECOVERY_STEP; + +@Configuration +@EnableBatchProcessing +public class RSAccessionRecoveryStepConfiguration { + @Autowired + @Qualifier(RS_ACCESSION_RECOVERY_SERVICE) + private RSAccessionRecoveryService RSAccessionRecoveryService; + + @Bean(RS_ACCESSION_RECOVERY_STEP) + public Step monotonicAccessionRecoveryAgentCategoryRSStep(StepBuilderFactory stepBuilderFactory) { + return stepBuilderFactory.get(RS_ACCESSION_RECOVERY_STEP) + .tasklet((contribution, chunkContext) -> { + RSAccessionRecoveryService.runRecoveryForCategoryRS(); + return null; + }) + .build(); + } +} diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java index e34c4bebe..3da450c20 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/ClusteringVariantJobConfigurationTest.java @@ -73,11 +73,13 @@ import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_NON_CLUSTERED_VARIANTS_FROM_MONGO_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_MERGE_CANDIDATES_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_RS_SPLIT_CANDIDATES_STEP; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_ACCESSION_RECOVERY_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.STUDY_CLUSTERING_STEP; import static uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration.MERGE_CANDIDATE_ID_PREFIX; import static uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration.SPLIT_CANDIDATE_ID_PREFIX; import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_MONGO; import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_VCF; +import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_RS_ACCESSION_RECOVERY; import static uk.ac.ebi.eva.accession.clustering.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_STUDY_FROM_MONGO; @RunWith(SpringRunner.class) @@ -99,6 +101,10 @@ public class ClusteringVariantJobConfigurationTest { @Qualifier(JOB_LAUNCHER_STUDY_FROM_MONGO) private JobLauncherTestUtils jobLauncherTestUtilsStudyFromMongo; + @Autowired + @Qualifier(JOB_LAUNCHER_RS_ACCESSION_RECOVERY) + private JobLauncherTestUtils jobLauncherTestUtilsMonotonicAccessionRecoveryAgent; + @Autowired private MongoTemplate mongoTemplate; @@ -219,6 +225,16 @@ public void assertDataThatExceedsChunkSizeIsFullyProcessed() throws Exception { .findFirst().get().getReadCount()); } + @Test + @DirtiesContext + public void testJobRSAccessionRecovery() throws Exception { + JobExecution jobExecution = jobLauncherTestUtilsMonotonicAccessionRecoveryAgent.launchJob(); + List expectedSteps = new ArrayList<>(); + expectedSteps.add(RS_ACCESSION_RECOVERY_STEP); + assertStepsExecuted(expectedSteps, jobExecution); + assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + } + private void createMergeCandidateEntriesThatExceedChunkSize(int numSplitCandidateOperations) { //Candidates for merge are entries with same locus but different RS SubmittedVariantInactiveEntity ss1 = new SubmittedVariantInactiveEntity( diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryTest.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryTest.java new file mode 100644 index 000000000..dfbbfa08f --- /dev/null +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/configuration/batch/jobs/RSAccessionRecoveryTest.java @@ -0,0 +1,223 @@ +package uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs; + +/* + * Copyright 2014-2018 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.batch.core.BatchStatus; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.test.JobLauncherTestUtils; +import org.springframework.batch.test.context.SpringBatchTest; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; +import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities.ContiguousIdBlock; +import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.repositories.ContiguousIdBlockRepository; +import uk.ac.ebi.eva.accession.clustering.test.configuration.RSAccessionRecoveryTestConfiguration; +import uk.ac.ebi.eva.accession.core.model.ClusteredVariant; +import uk.ac.ebi.eva.accession.core.model.eva.ClusteredVariantEntity; +import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.ClusteredVariantAccessioningRepository; +import uk.ac.ebi.eva.commons.core.models.VariantType; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static uk.ac.ebi.eva.accession.clustering.test.configuration.RSAccessionRecoveryTestConfiguration.JOB_LAUNCHER_RS_ACCESSION_RECOVERY; + +@RunWith(SpringRunner.class) +@ContextConfiguration(classes = {RSAccessionRecoveryTestConfiguration.class,}) +@TestPropertySource("classpath:rs-accession-recovery.properties") +@SpringBatchTest +public class RSAccessionRecoveryTest { + @Autowired + private ContiguousIdBlockRepository blockRepository; + + @Autowired + private ClusteredVariantAccessioningRepository mongoRepository; + + @Autowired + @Qualifier(JOB_LAUNCHER_RS_ACCESSION_RECOVERY) + private JobLauncherTestUtils jobLauncherTestUtilsMonotonicAccessionRecoveryAgent; + + @Test + @DirtiesContext + public void testContiguousBlocksForCategoryRSAreRecovered() throws Exception { + initializeMongoDbWithUncommittedAccessions(); + verifyInitialDBState(); + + // recovery cut off time is -14 days (provided in rs-accession-recovery.properties) + JobExecution jobExecution = jobLauncherTestUtilsMonotonicAccessionRecoveryAgent.launchJob(); + assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + + verifyEndDBState(); + } + + private void initializeMongoDbWithUncommittedAccessions() { + mongoRepository.deleteAll(); + + List clusteredVariantEntityList = new ArrayList<>(); + // Entries for 1st block + for (long i = 3000000000l; i <= 3000000029l; i++) { + ClusteredVariant model = new ClusteredVariant("assembly", 1111, + "contig", 100, VariantType.SNV, false, LocalDateTime.now()); + ClusteredVariantEntity entity = new ClusteredVariantEntity(i, "hash" + i, model, 1); + clusteredVariantEntityList.add(entity); + } + + // Entries for 2nd block - Missing 5 RS (3000000035l - 3000000039l) + for (long i = 3000000030l; i <= 3000000034l; i++) { + ClusteredVariant model = new ClusteredVariant("assembly", 1111, + "contig", 100, VariantType.SNV, false, LocalDateTime.now()); + ClusteredVariantEntity entity = new ClusteredVariantEntity(i, "hash" + i, model, 1); + clusteredVariantEntityList.add(entity); + } + for (long i = 3000000040l; i <= 3000000059l; i++) { + ClusteredVariant model = new ClusteredVariant("assembly", 1111, + "contig", 100, VariantType.SNV, false, LocalDateTime.now()); + ClusteredVariantEntity entity = new ClusteredVariantEntity(i, "hash" + i, model, 1); + clusteredVariantEntityList.add(entity); + } + + // Entries for 3rd block + for (long i = 3000000060l; i <= 3000000089l; i++) { + ClusteredVariant model = new ClusteredVariant("assembly", 1111, + "contig", 100, VariantType.SNV, false, LocalDateTime.now()); + ClusteredVariantEntity entity = new ClusteredVariantEntity(i, "hash" + i, model, 1); + clusteredVariantEntityList.add(entity); + } + + // Entries for 5th block + for (long i = 3000000120l; i <= 3000000129l; i++) { + ClusteredVariant model = new ClusteredVariant("assembly", 1111, + "contig", 100, VariantType.SNV, false, LocalDateTime.now()); + ClusteredVariantEntity entity = new ClusteredVariantEntity(i, "hash" + i, model, 1); + clusteredVariantEntityList.add(entity); + } + + mongoRepository.saveAll(clusteredVariantEntityList); + } + + private void verifyInitialDBState() { + // Initial state of Contiguous Id Block DB is 5 blocks are present but their "last_committed" is not updated + // (Initialized using "resources/test-data/rs_accession_recovery_test_data.sql") + + // block id first value last value last committed reserved last_updated_timestamp | remarks + // 1 3000000000 3000000029 2999999999 true 1970-01-01 00:00:00 | should be recovered + // 2 3000000030 3000000059 3000000029 true 1970-01-01 00:00:00 | should be recovered + // 3 3000000060 3000000089 3000000059 true 1970-01-01 00:00:00 | should be recovered + // 4 3000000090 3000000119 3000000089 true 1970-01-01 00:00:00 | should be recovered + // 5 3000000120 3000000149 3000000119 true 2099-01-01 00:00:00 | should not be recovered + + // Mongo DB + // 95 accessions have been used in mongoDB but are not reflected in the block allocation table + // 30 accessions belong to 1st block (3000000000 to 3000000029), + // 25 to the 2nd block (3000000030 to 300000034 and 3000000040 to 3000000059) + // 30 to the 3rd block (3000000060 to 3000000089) + // None in 4th block + // 10 to the 5th block (3000000120 to 3000000129) + + assertEquals(95, mongoRepository.count()); // 30 + 25 + 30 + 10 + assertEquals(5, blockRepository.count()); + + // Since none of the 4 blocks got committed - everyone's last committed value is first_value - 1 + ContiguousIdBlock block1 = blockRepository.findById(1l).get(); + assertEquals(3000000000l, block1.getFirstValue()); + assertEquals(2999999999l, block1.getLastCommitted()); + assertEquals(3000000029l, block1.getLastValue()); + assertEquals("test-instance-recover-state-00", block1.getApplicationInstanceId()); + assertTrue(block1.isReserved()); + + ContiguousIdBlock block2 = blockRepository.findById(2l).get(); + assertEquals(3000000030l, block2.getFirstValue()); + assertEquals(3000000029l, block2.getLastCommitted()); + assertEquals(3000000059l, block2.getLastValue()); + assertEquals("test-instance-recover-state-00", block2.getApplicationInstanceId()); + assertTrue(block2.isReserved()); + + ContiguousIdBlock block3 = blockRepository.findById(3l).get(); + assertEquals(3000000060l, block3.getFirstValue()); + assertEquals(3000000059l, block3.getLastCommitted()); + assertEquals(3000000089l, block3.getLastValue()); + assertEquals("test-instance-recover-state-00", block3.getApplicationInstanceId()); + assertTrue(block3.isReserved()); + + ContiguousIdBlock block4 = blockRepository.findById(4l).get(); + assertEquals(3000000090l, block4.getFirstValue()); + assertEquals(3000000089l, block4.getLastCommitted()); + assertEquals(3000000119l, block4.getLastValue()); + assertEquals("test-instance-recover-state-00", block4.getApplicationInstanceId()); + assertTrue(block4.isReserved()); + + ContiguousIdBlock block5 = blockRepository.findById(5l).get(); + assertEquals(3000000120l, block5.getFirstValue()); + assertEquals(3000000119l, block5.getLastCommitted()); + assertEquals(3000000149l, block5.getLastValue()); + assertEquals("test-instance-recover-state-00", block5.getApplicationInstanceId()); + assertTrue(block5.isReserved()); + } + + private void verifyEndDBState() { + assertEquals(5, blockRepository.count()); + + // Block Recovered - recovered entire block + ContiguousIdBlock block1 = blockRepository.findById(1l).get(); + assertEquals(3000000000l, block1.getFirstValue()); + assertEquals(3000000029l, block1.getLastCommitted()); + assertEquals(3000000029l, block1.getLastValue()); + assertEquals("0", block1.getApplicationInstanceId()); + assertTrue(block1.isNotReserved()); + + // Block Recovered partially - (used 3000000030-3000000034 and 3000000040-3000000059) + // since there are unused accessions(3000000035-3000000039), last_committed is updated to 3000000034 only) + ContiguousIdBlock block2 = blockRepository.findById(2l).get(); + assertEquals(3000000030l, block2.getFirstValue()); + assertEquals(3000000034l, block2.getLastCommitted()); + assertEquals(3000000059l, block2.getLastValue()); + assertEquals("0", block2.getApplicationInstanceId()); + assertTrue(block2.isNotReserved()); + + // Block Recovered - recovered entire block + ContiguousIdBlock block3 = blockRepository.findById(3l).get(); + assertEquals(3000000060l, block3.getFirstValue()); + assertEquals(3000000089l, block3.getLastCommitted()); + assertEquals(3000000089l, block3.getLastValue()); + assertEquals("0", block3.getApplicationInstanceId()); + assertTrue(block3.isNotReserved()); + + // Block Recovered - None of the accessions are used, just released the block + ContiguousIdBlock block4 = blockRepository.findById(4l).get(); + assertEquals(3000000090l, block4.getFirstValue()); + assertEquals(3000000089l, block4.getLastCommitted()); + assertEquals(3000000119l, block4.getLastValue()); + assertEquals("0", block4.getApplicationInstanceId()); + assertTrue(block4.isNotReserved()); + + // Block Not Recovered - Block not to be recovered as it's last update is after cut off time + ContiguousIdBlock block5 = blockRepository.findById(5l).get(); + assertEquals(3000000120l, block5.getFirstValue()); + assertEquals(3000000119l, block5.getLastCommitted()); + assertEquals(3000000149l, block5.getLastValue()); + assertEquals("test-instance-recover-state-00", block5.getApplicationInstanceId()); + assertTrue(block5.isReserved()); + } +} \ No newline at end of file diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java index e57c5e832..8a294efb1 100644 --- a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/BatchTestConfiguration.java @@ -27,29 +27,42 @@ import org.springframework.context.annotation.Import; import org.springframework.core.io.ResourceLoader; import org.springframework.transaction.PlatformTransactionManager; - -import uk.ac.ebi.eva.accession.clustering.batch.listeners.JobExecutionSetter; -import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.TargetSSReaderForBackPropRSConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.BackPropagatedRSWriterConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.ClusteringMongoReaderConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.ClusteringWriterConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitCandidatesReaderConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.RSMergeAndSplitWriterConfiguration; +import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.TargetSSReaderForBackPropRSConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.io.VcfReaderConfiguration; -import uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs.*; +import uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs.BackPropagateRSJobConfiguration; +import uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs.ClusterUnclusteredVariantsJobConfiguration; +import uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs.ClusteringFromMongoJobConfiguration; +import uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs.ClusteringFromVcfJobConfiguration; +import uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs.ProcessRemappedVariantsWithRSJobConfiguration; +import uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs.RSAccessionRecoveryJobConfiguration; +import uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs.ResolveMergeThenSplitCandidatesJobConfiguration; +import uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs.StudyClusteringJobConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs.qc.NewClusteredVariantsQCJobConfiguration; +import uk.ac.ebi.eva.accession.clustering.configuration.batch.listeners.JobExecutionSetterConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.listeners.ListenersConfiguration; +import uk.ac.ebi.eva.accession.clustering.configuration.batch.listeners.RSAccessionRecoveryJobListenerConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.policies.ChunkSizeCompletionPolicyConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.processors.ClusteringVariantProcessorConfiguration; +import uk.ac.ebi.eva.accession.clustering.configuration.batch.recovery.RSAccessionRecoveryServiceConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.steps.AccessioningShutdownStepConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.steps.ClusteringFromMongoStepConfiguration; import uk.ac.ebi.eva.accession.clustering.configuration.batch.steps.ClusteringFromVcfStepConfiguration; +import uk.ac.ebi.eva.accession.clustering.configuration.batch.steps.RSAccessionRecoveryStepConfiguration; import uk.ac.ebi.eva.accession.clustering.runner.ClusteringCommandLineRunner; import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; import javax.sql.DataSource; -import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.*; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_FROM_MONGO_JOB; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.CLUSTERING_FROM_VCF_JOB; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.PROCESS_REMAPPED_VARIANTS_WITH_RS_JOB; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_ACCESSION_RECOVERY_JOB; +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.STUDY_CLUSTERING_JOB; import static uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs.qc.NewClusteredVariantsQCJobConfiguration.NEW_CLUSTERED_VARIANTS_QC_JOB; @EnableAutoConfiguration @@ -75,7 +88,11 @@ ClusteringCommandLineRunner.class, ChunkSizeCompletionPolicyConfiguration.class, AccessioningShutdownStepConfiguration.class, - JobExecutionSetter.class}) + JobExecutionSetterConfiguration.class, + RSAccessionRecoveryJobConfiguration.class, + RSAccessionRecoveryStepConfiguration.class, + RSAccessionRecoveryServiceConfiguration.class, + RSAccessionRecoveryJobListenerConfiguration.class}) public class BatchTestConfiguration { public static final String JOB_LAUNCHER_FROM_VCF = "JOB_LAUNCHER_FROM_VCF"; @@ -88,6 +105,8 @@ public class BatchTestConfiguration { public static final String JOB_LAUNCHER_FROM_MONGO_ONLY_FIRST_STEP = "JOB_LAUNCHER_FROM_MONGO_ONLY_FIRST_STEP"; + public static final String JOB_LAUNCHER_RS_ACCESSION_RECOVERY = "JOB_LAUNCHER_RS_ACCESSION_RECOVERY"; + @Autowired private BatchProperties properties; @@ -136,6 +155,18 @@ public void setJob(@Qualifier(STUDY_CLUSTERING_JOB) Job job) { }; } + @Bean(JOB_LAUNCHER_RS_ACCESSION_RECOVERY) + public JobLauncherTestUtils jobLauncherTestUtilsRSAccessionRecoveryJob() { + + return new JobLauncherTestUtils() { + @Override + @Autowired + public void setJob(@Qualifier(RS_ACCESSION_RECOVERY_JOB) Job job) { + super.setJob(job); + } + }; + } + @Bean(JOB_LAUNCHER_NEW_CLUSTERED_VARIANTS_QC) public JobLauncherTestUtils jobLauncherTestUtilsNewClusteredVariants() { diff --git a/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/RSAccessionRecoveryTestConfiguration.java b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/RSAccessionRecoveryTestConfiguration.java new file mode 100644 index 000000000..b1bdcf1ab --- /dev/null +++ b/eva-accession-clustering/src/test/java/uk/ac/ebi/eva/accession/clustering/test/configuration/RSAccessionRecoveryTestConfiguration.java @@ -0,0 +1,81 @@ +/* + * Copyright 2018 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.ac.ebi.eva.accession.clustering.test.configuration; + +import org.springframework.batch.core.Job; +import org.springframework.batch.core.configuration.annotation.BatchConfigurer; +import org.springframework.batch.test.JobLauncherTestUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.clustering.configuration.batch.jobs.RSAccessionRecoveryJobConfiguration; +import uk.ac.ebi.eva.accession.clustering.configuration.batch.listeners.RSAccessionRecoveryJobListenerConfiguration; +import uk.ac.ebi.eva.accession.clustering.configuration.batch.recovery.RSAccessionRecoveryServiceConfiguration; +import uk.ac.ebi.eva.accession.clustering.configuration.batch.steps.RSAccessionRecoveryStepConfiguration; +import uk.ac.ebi.eva.accession.core.configuration.nonhuman.ClusteredVariantAccessioningConfiguration; +import uk.ac.ebi.eva.commons.batch.configuration.SpringBoot1CompatibilityConfiguration; +import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; + +import javax.persistence.EntityManagerFactory; +import javax.sql.DataSource; + +import static uk.ac.ebi.eva.accession.clustering.configuration.BeanNames.RS_ACCESSION_RECOVERY_JOB; + +@EnableAutoConfiguration +@Import({RSAccessionRecoveryJobConfiguration.class, + RSAccessionRecoveryStepConfiguration.class, + RSAccessionRecoveryServiceConfiguration.class, + RSAccessionRecoveryJobListenerConfiguration.class, + ClusteredVariantAccessioningConfiguration.class +}) +public class RSAccessionRecoveryTestConfiguration { + public static final String JOB_LAUNCHER_RS_ACCESSION_RECOVERY = "JOB_LAUNCHER_RS_ACCESSION_RECOVERY"; + + @Bean + public BatchConfigurer configurer(DataSource dataSource, EntityManagerFactory entityManagerFactory) + throws Exception { + return SpringBoot1CompatibilityConfiguration.getSpringBoot1CompatibleBatchConfigurer(dataSource, + entityManagerFactory); + } + + @Bean + public JobLauncherTestUtils jobLauncherTestUtils(BatchConfigurer configurer) throws Exception { + JobLauncherTestUtils jobLauncherTestUtils = new JobLauncherTestUtils(); + jobLauncherTestUtils.setJobLauncher(configurer.getJobLauncher()); + jobLauncherTestUtils.setJobRepository(configurer.getJobRepository()); + return jobLauncherTestUtils; + } + + @Bean + public JobExecutionApplicationListener jobExecutionApplicationListener() { + return new JobExecutionApplicationListener(); + } + + @Bean(JOB_LAUNCHER_RS_ACCESSION_RECOVERY) + public JobLauncherTestUtils jobLauncherTestUtilsMonotonicAccessionRecoveryAgent() { + + return new JobLauncherTestUtils() { + @Override + @Autowired + public void setJob(@Qualifier(RS_ACCESSION_RECOVERY_JOB) Job job) { + super.setJob(job); + } + }; + } +} \ No newline at end of file diff --git a/eva-accession-clustering/src/test/resources/backpropagation-test.properties b/eva-accession-clustering/src/test/resources/backpropagation-test.properties index d24b6632d..3c24d4c6d 100644 --- a/eva-accession-clustering/src/test/resources/backpropagation-test.properties +++ b/eva-accession-clustering/src/test/resources/backpropagation-test.properties @@ -1,3 +1,4 @@ +recovery.cutoff.days=14 parameters.vcf=src/test/resources/input-files/vcf/aggregated_accessioned.vcf.gz parameters.projectAccession=projectId_1 parameters.projects=projectId_2,projectId_3 diff --git a/eva-accession-clustering/src/test/resources/clustering-issuance-test.properties b/eva-accession-clustering/src/test/resources/clustering-issuance-test.properties index 03f78b631..e4ac2f7a5 100644 --- a/eva-accession-clustering/src/test/resources/clustering-issuance-test.properties +++ b/eva-accession-clustering/src/test/resources/clustering-issuance-test.properties @@ -1,3 +1,4 @@ +recovery.cutoff.days=14 parameters.vcf=src/test/resources/input-files/vcf/aggregated_accessioned.vcf.gz parameters.projectAccession=projectId_1 parameters.projects=projectId_2,projectId_3 diff --git a/eva-accession-clustering/src/test/resources/clustering-pipeline-test.properties b/eva-accession-clustering/src/test/resources/clustering-pipeline-test.properties index 07ec7ca23..e11c5f099 100644 --- a/eva-accession-clustering/src/test/resources/clustering-pipeline-test.properties +++ b/eva-accession-clustering/src/test/resources/clustering-pipeline-test.properties @@ -1,3 +1,4 @@ +recovery.cutoff.days=14 parameters.vcf=src/test/resources/input-files/vcf/aggregated_accessioned.vcf.gz parameters.projectAccession=projectId_1 parameters.projects=projectId_2,projectId_3 diff --git a/eva-accession-clustering/src/test/resources/clustering-qc-test.properties b/eva-accession-clustering/src/test/resources/clustering-qc-test.properties index 2bac56f82..35fad5052 100644 --- a/eva-accession-clustering/src/test/resources/clustering-qc-test.properties +++ b/eva-accession-clustering/src/test/resources/clustering-qc-test.properties @@ -1,3 +1,4 @@ +recovery.cutoff.days=14 parameters.vcf=src/test/resources/input-files/vcf/aggregated_accessioned.vcf.gz parameters.projectAccession=projectId_1 parameters.projects=projectId_2,projectId_3 diff --git a/eva-accession-clustering/src/test/resources/clustering-writer-test.properties b/eva-accession-clustering/src/test/resources/clustering-writer-test.properties index 448271e33..30cb01327 100644 --- a/eva-accession-clustering/src/test/resources/clustering-writer-test.properties +++ b/eva-accession-clustering/src/test/resources/clustering-writer-test.properties @@ -1,3 +1,5 @@ +recovery.cutoff.days=14 + parameters.vcf=src/test/resources/input-files/vcf/aggregated_accessioned.vcf.gz parameters.projectAccession=projectId_1 parameters.projects=projectId_2,projectId_3 diff --git a/eva-accession-clustering/src/test/resources/merge-split-test.properties b/eva-accession-clustering/src/test/resources/merge-split-test.properties index 5cb0d19a1..8740229b5 100644 --- a/eva-accession-clustering/src/test/resources/merge-split-test.properties +++ b/eva-accession-clustering/src/test/resources/merge-split-test.properties @@ -1,3 +1,4 @@ +recovery.cutoff.days=14 parameters.vcf=src/test/resources/input-files/vcf/aggregated_accessioned.vcf.gz parameters.projectAccession=projectId_1 parameters.assemblyAccession=GCA_000000001.1 diff --git a/eva-accession-clustering/src/test/resources/output-files/rsReport.txt b/eva-accession-clustering/src/test/resources/output-files/rsReport.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/eva-accession-clustering/src/test/resources/rs-accession-recovery.properties b/eva-accession-clustering/src/test/resources/rs-accession-recovery.properties new file mode 100644 index 000000000..a66292426 --- /dev/null +++ b/eva-accession-clustering/src/test/resources/rs-accession-recovery.properties @@ -0,0 +1,43 @@ +recovery.cutoff.days=14 + +spring.datasource.driver-class-name=org.hsqldb.jdbcDriver +spring.datasource.url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +spring.datasource.username=SA +spring.datasource.password= +spring.datasource.schema=test-data/contiguous_id_blocks_schema.sql +spring.datasource.data=test-data/rs_accession_recovery_test_data.sql +spring.jpa.hibernate.ddl-auto=update + +parameters.vcf=src/test/resources/input-files/vcf/aggregated_accessioned.vcf.gz +parameters.projectAccession=projectId_1 +parameters.projects=projectId_2,projectId_3 +parameters.assemblyAccession=asm2 +parameters.remappedFrom=asm1 +parameters.rsReportPath=src/test/resources/output-files/rsReport.txt +parameters.chunkSize=2 + +eva.count-stats.url=http://localhost:8080 +eva.count-stats.username=username +eva.count-stats.password=password + +accessioning.submitted.categoryId=ss +accessioning.clustered.categoryId=rs + +accessioning.monotonic.ss.blockSize=100000 +accessioning.monotonic.ss.blockStartValue=5000000000 +accessioning.monotonic.ss.nextBlockInterval=1000000000 +accessioning.monotonic.rs.blockSize=100000 +accessioning.monotonic.rs.blockStartValue=3000000000 +accessioning.monotonic.rs.nextBlockInterval=1000000000 + +spring.data.mongodb.database=test-db +spring.data.mongodb.host=|eva.mongo.host.test| +spring.data.mongodb.password= +spring.data.mongodb.port=27017 +mongodb.read-preference=primary + +# See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding +spring.main.allow-bean-definition-overriding=true + +# TODO jmmut: without this, tests pass in mvn but not in intellij (in intellij pass individually, but not together) +spring.jmx.enabled=false \ No newline at end of file diff --git a/eva-accession-clustering/src/test/resources/test-data/contiguous_id_blocks_schema.sql b/eva-accession-clustering/src/test/resources/test-data/contiguous_id_blocks_schema.sql new file mode 100644 index 000000000..495f9a4fb --- /dev/null +++ b/eva-accession-clustering/src/test/resources/test-data/contiguous_id_blocks_schema.sql @@ -0,0 +1,10 @@ +CREATE TABLE contiguous_id_blocks ( + id bigint not NULL, + application_instance_id varchar(255) not NULL, + category_id varchar(255) not NULL, + first_value bigint not NULL, + last_committed bigint not NULL, + last_value bigint not NULL, + reserved boolean NOT NULL, + last_updated_timestamp timestamp NOT NULL +); diff --git a/eva-accession-clustering/src/test/resources/test-data/rs_accession_recovery_test_data.sql b/eva-accession-clustering/src/test/resources/test-data/rs_accession_recovery_test_data.sql new file mode 100644 index 000000000..0053e69cb --- /dev/null +++ b/eva-accession-clustering/src/test/resources/test-data/rs_accession_recovery_test_data.sql @@ -0,0 +1,14 @@ +INSERT INTO contiguous_id_blocks +VALUES (1, 'test-instance-recover-state-00', 'rs', 3000000000, 2999999999, 3000000029, true, '1970-01-01 00:00:00'); + +INSERT INTO contiguous_id_blocks +VALUES (2, 'test-instance-recover-state-00', 'rs', 3000000030, 3000000029, 3000000059, true, '1970-01-01 00:00:00'); + +INSERT INTO contiguous_id_blocks +VALUES (3, 'test-instance-recover-state-00', 'rs', 3000000060, 3000000059, 3000000089, true, '1970-01-01 00:00:00'); + +INSERT INTO contiguous_id_blocks +VALUES (4, 'test-instance-recover-state-00', 'rs', 3000000090, 3000000089, 3000000119, true, '1970-01-01 00:00:00'); + +INSERT INTO contiguous_id_blocks +VALUES (5, 'test-instance-recover-state-00', 'rs', 3000000120, 3000000119, 3000000149, true, '2099-01-01 00:00:00'); \ No newline at end of file diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/recovery/SSAccessionRecoveryService.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/recovery/SSAccessionRecoveryService.java new file mode 100644 index 000000000..cecc61101 --- /dev/null +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/batch/recovery/SSAccessionRecoveryService.java @@ -0,0 +1,28 @@ +package uk.ac.ebi.eva.accession.pipeline.batch.recovery; + +import org.springframework.batch.core.JobExecution; +import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicAccessionRecoveryAgent; + +import java.time.LocalDateTime; + +public class SSAccessionRecoveryService { + private final static String CATEGORY_ID = "ss"; + private MonotonicAccessionRecoveryAgent monotonicAccessionRecoveryAgent; + private JobExecution jobExecution; + private long recoveryCutOffDays; + + public SSAccessionRecoveryService(MonotonicAccessionRecoveryAgent monotonicAccessionRecoveryAgent, + long recoveryCutOffDays) { + this.monotonicAccessionRecoveryAgent = monotonicAccessionRecoveryAgent; + this.recoveryCutOffDays = recoveryCutOffDays; + } + + public void runRecoveryForCategorySS() { + LocalDateTime recoveryCutOffTime = LocalDateTime.now().minusDays(recoveryCutOffDays); + monotonicAccessionRecoveryAgent.runRecovery(CATEGORY_ID, jobExecution.getJobId().toString(), recoveryCutOffTime); + } + + public void setJobExecution(JobExecution jobExecution) { + this.jobExecution = jobExecution; + } +} diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/BeanNames.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/BeanNames.java index a1961af43..39c16110f 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/BeanNames.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/BeanNames.java @@ -39,4 +39,12 @@ public class BeanNames { public static final String ACCESSIONING_SHUTDOWN_STEP = "ACCESSIONING_SHUTDOWN_STEP"; public static final String SUBSNP_ACCESSION_JOB_LISTENER = "SUBSNP_ACCESSION_JOB_LISTENER"; + + public static final String SS_ACCESSION_RECOVERY_SERVICE = "SS_ACCESSION_RECOVERY_SERVICE"; + + public static final String SS_ACCESSION_RECOVERY_STEP = "SS_ACCESSION_RECOVERY_STEP"; + + public static final String SS_ACCESSION_RECOVERY_JOB = "SS_ACCESSION_RECOVERY_JOB"; + + public static final String SS_ACCESSION_RECOVERY_JOB_LISTENER = "SS_ACCESSION_RECOVERY_JOB_LISTENER"; } diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SSAccessionRecoveryJobConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SSAccessionRecoveryJobConfiguration.java new file mode 100644 index 000000000..25a2669d2 --- /dev/null +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/SSAccessionRecoveryJobConfiguration.java @@ -0,0 +1,39 @@ +package uk.ac.ebi.eva.accession.pipeline.configuration.batch.jobs; + +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobExecutionListener; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SS_ACCESSION_RECOVERY_JOB; +import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SS_ACCESSION_RECOVERY_JOB_LISTENER; +import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SS_ACCESSION_RECOVERY_STEP; + +@Configuration +@EnableBatchProcessing +public class SSAccessionRecoveryJobConfiguration { + + @Autowired + @Qualifier(SS_ACCESSION_RECOVERY_STEP) + private Step ssAccessionRecoveryStep; + + @Autowired + @Qualifier(SS_ACCESSION_RECOVERY_JOB_LISTENER) + private JobExecutionListener ssAccessionRecoveryJobListener; + + @Bean(SS_ACCESSION_RECOVERY_JOB) + public Job createSSAccessionRecoveryJob(JobBuilderFactory jobBuilderFactory) { + return jobBuilderFactory.get(SS_ACCESSION_RECOVERY_JOB) + .incrementer(new RunIdIncrementer()) + .start(ssAccessionRecoveryStep) + .listener(ssAccessionRecoveryJobListener) + .build(); + } + +} diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/listeners/SSAccessionRecoveryJobListenerConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/listeners/SSAccessionRecoveryJobListenerConfiguration.java new file mode 100644 index 000000000..fdc14b549 --- /dev/null +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/listeners/SSAccessionRecoveryJobListenerConfiguration.java @@ -0,0 +1,29 @@ +package uk.ac.ebi.eva.accession.pipeline.configuration.batch.listeners; + +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobExecutionListener; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import uk.ac.ebi.eva.accession.pipeline.batch.recovery.SSAccessionRecoveryService; + +import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SS_ACCESSION_RECOVERY_JOB_LISTENER; +import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SS_ACCESSION_RECOVERY_SERVICE; + +@Configuration +public class SSAccessionRecoveryJobListenerConfiguration { + @Bean(SS_ACCESSION_RECOVERY_JOB_LISTENER) + public JobExecutionListener jobExecutionListener(@Qualifier(SS_ACCESSION_RECOVERY_SERVICE) + SSAccessionRecoveryService SSAccessionRecoveryService) { + return new JobExecutionListener() { + @Override + public void beforeJob(JobExecution jobExecution) { + SSAccessionRecoveryService.setJobExecution(jobExecution); + } + + @Override + public void afterJob(JobExecution jobExecution) { + } + }; + } +} diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/listeners/SubsnpAccessionJobExecutionListener.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/listeners/SubsnpAccessionJobExecutionListener.java index b4e2a53e0..8bcac4cc1 100644 --- a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/listeners/SubsnpAccessionJobExecutionListener.java +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/listeners/SubsnpAccessionJobExecutionListener.java @@ -5,6 +5,7 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import uk.ac.ebi.eva.accession.core.service.nonhuman.SubmittedVariantAccessioningService; import uk.ac.ebi.eva.accession.pipeline.batch.io.AccessionWriter; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.ACCESSION_WRITER; @@ -14,7 +15,8 @@ public class SubsnpAccessionJobExecutionListener { @Bean(SUBSNP_ACCESSION_JOB_LISTENER) - public JobExecutionListener jobExecutionListener(@Qualifier(ACCESSION_WRITER) AccessionWriter accessionWriter) { + public JobExecutionListener jobExecutionListener(@Qualifier(ACCESSION_WRITER) AccessionWriter accessionWriter, + SubmittedVariantAccessioningService submittedVariantAccessioningService) { return new JobExecutionListener() { @Override public void beforeJob(JobExecution jobExecution) { @@ -23,7 +25,7 @@ public void beforeJob(JobExecution jobExecution) { @Override public void afterJob(JobExecution jobExecution) { - // Do nothing + submittedVariantAccessioningService.shutDownAccessionGenerator(); } }; } diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/recovery/SSAccessionRecoveryServiceConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/recovery/SSAccessionRecoveryServiceConfiguration.java new file mode 100644 index 000000000..7b6bce345 --- /dev/null +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/recovery/SSAccessionRecoveryServiceConfiguration.java @@ -0,0 +1,32 @@ +package uk.ac.ebi.eva.accession.pipeline.configuration.batch.recovery; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import uk.ac.ebi.ampt2d.commons.accession.generators.monotonic.MonotonicAccessionRecoveryAgent; +import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.service.ContiguousIdBlockService; +import uk.ac.ebi.eva.accession.core.service.nonhuman.eva.SubmittedVariantAccessioningDatabaseService; +import uk.ac.ebi.eva.accession.pipeline.batch.recovery.SSAccessionRecoveryService; + +import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SS_ACCESSION_RECOVERY_SERVICE; + +@Configuration +public class SSAccessionRecoveryServiceConfiguration { + @Autowired + private ContiguousIdBlockService blockService; + @Autowired + private SubmittedVariantAccessioningDatabaseService submittedVariantAccessioningDatabaseService; + + @Value("${recovery.cutoff.days}") + private long recoveryCutOffDays; + + @Bean(SS_ACCESSION_RECOVERY_SERVICE) + public SSAccessionRecoveryService getSSAccessionRecoveryService() { + return new SSAccessionRecoveryService(getMonotonicAccessionRecoveryAgent(), recoveryCutOffDays); + } + + private MonotonicAccessionRecoveryAgent getMonotonicAccessionRecoveryAgent() { + return new MonotonicAccessionRecoveryAgent(blockService, submittedVariantAccessioningDatabaseService); + } +} diff --git a/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SSAccessionRecoveryStepConfiguration.java b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SSAccessionRecoveryStepConfiguration.java new file mode 100644 index 000000000..758d75616 --- /dev/null +++ b/eva-accession-pipeline/src/main/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/SSAccessionRecoveryStepConfiguration.java @@ -0,0 +1,31 @@ +package uk.ac.ebi.eva.accession.pipeline.configuration.batch.steps; + +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import uk.ac.ebi.eva.accession.pipeline.batch.recovery.SSAccessionRecoveryService; + +import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SS_ACCESSION_RECOVERY_SERVICE; +import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SS_ACCESSION_RECOVERY_STEP; + +@Configuration +@EnableBatchProcessing +public class SSAccessionRecoveryStepConfiguration { + @Autowired + @Qualifier(SS_ACCESSION_RECOVERY_SERVICE) + private SSAccessionRecoveryService SSAccessionRecoveryService; + + @Bean(SS_ACCESSION_RECOVERY_STEP) + public Step ssAccessionRecoveryStep(StepBuilderFactory stepBuilderFactory) { + return stepBuilderFactory.get(SS_ACCESSION_RECOVERY_STEP) + .tasklet((contribution, chunkContext) -> { + SSAccessionRecoveryService.runRecoveryForCategorySS(); + return null; + }) + .build(); + } +} diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java index 52adf8af5..6dffc1dd8 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsRecoverStateTest.java @@ -182,21 +182,21 @@ private void verifyInitialDBState() { assertEquals(5000000030l, block2.getFirstValue()); assertEquals(5000000029l, block2.getLastCommitted()); assertEquals(5000000059l, block2.getLastValue()); - assertEquals("test-instance-recover-state-00", block1.getApplicationInstanceId()); + assertEquals("test-instance-recover-state-00", block2.getApplicationInstanceId()); assertTrue(block2.isNotReserved()); ContiguousIdBlock block3 = blockRepository.findById(3l).get(); assertEquals(5000000060l, block3.getFirstValue()); assertEquals(5000000059l, block3.getLastCommitted()); assertEquals(5000000089l, block3.getLastValue()); - assertEquals("test-instance-recover-state-00", block1.getApplicationInstanceId()); + assertEquals("test-instance-recover-state-00", block3.getApplicationInstanceId()); assertTrue(block3.isNotReserved()); ContiguousIdBlock block4 = blockRepository.findById(4l).get(); assertEquals(5000000090l, block4.getFirstValue()); assertEquals(5000000089l, block4.getLastCommitted()); assertEquals(5000000119l, block4.getLastValue()); - assertEquals("test-instance-recover-state-00", block1.getApplicationInstanceId()); + assertEquals("test-instance-recover-state-00", block4.getApplicationInstanceId()); assertTrue(block4.isNotReserved()); } @@ -223,7 +223,7 @@ private void verifyEndDBState() { assertEquals(5000000030l, block2.getFirstValue()); assertEquals(5000000059l, block2.getLastCommitted()); assertEquals(5000000059l, block2.getLastValue()); - assertEquals("0", block1.getApplicationInstanceId()); + assertEquals("0", block2.getApplicationInstanceId()); assertTrue(block2.isNotReserved()); // Block Recovered - (No accession used from this block as entire block was already used) @@ -231,7 +231,7 @@ private void verifyEndDBState() { assertEquals(5000000060l, block3.getFirstValue()); assertEquals(5000000089l, block3.getLastCommitted()); assertEquals(5000000089l, block3.getLastValue()); - assertEquals("0", block1.getApplicationInstanceId()); + assertEquals("0", block3.getApplicationInstanceId()); assertTrue(block3.isNotReserved()); // used the remaining 17 (22 - 5 (2nd block)) from 4th block @@ -239,7 +239,7 @@ private void verifyEndDBState() { assertEquals(5000000090l, block4.getFirstValue()); assertEquals(5000000106l, block4.getLastCommitted()); assertEquals(5000000119l, block4.getLastValue()); - assertEquals("0", block1.getApplicationInstanceId()); + assertEquals("0", block4.getApplicationInstanceId()); assertTrue(block4.isNotReserved()); } diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/JobFailureBlocksReleasedTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/JobFailureBlocksReleasedTest.java new file mode 100644 index 000000000..b5e7feaf9 --- /dev/null +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/JobFailureBlocksReleasedTest.java @@ -0,0 +1,119 @@ +package uk.ac.ebi.eva.accession.pipeline.runner; + +/* + * Copyright 2014-2018 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.batch.test.context.SpringBatchTest; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; +import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.repositories.ContiguousIdBlockRepository; +import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; +import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; +import uk.ac.ebi.eva.commons.batch.io.VcfReader; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.CREATE_SUBSNP_ACCESSION_JOB; +import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.deleteTemporaryContigAndVariantFiles; +import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.getOriginalVcfContent; +import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.injectErrorIntoTempVcf; +import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.useOriginalVcfFile; +import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.useTempVcfFile; +import static uk.ac.ebi.eva.accession.pipeline.runner.RunnerUtil.writeToTempVCFFile; + +@RunWith(SpringRunner.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class}) +@TestPropertySource("classpath:accession-pipeline-test.properties") +@SpringBatchTest +public class JobFailureBlocksReleasedTest { + @Autowired + private InputParameters inputParameters; + + @Autowired + private EvaAccessionJobLauncherCommandLineRunner runner; + + @Autowired + private VcfReader vcfReader; + + @Autowired + private MongoTemplate mongoTemplate; + + private static File tempVcfInputFileToTestFailingJobs; + + private static Path tempVcfOutputDir; + + private static String originalVcfInputFilePath; + + private static String originalVcfContent; + + @Autowired + private ContiguousIdBlockRepository blockRepository; + + @BeforeClass + public static void initializeTempFile() throws Exception { + tempVcfInputFileToTestFailingJobs = File.createTempFile("resumeFailingJob", ".vcf.gz"); + tempVcfOutputDir = Files.createTempDirectory("contigs_variants_dir"); + } + + @AfterClass + public static void deleteTempFile() { + tempVcfInputFileToTestFailingJobs.delete(); + } + + @Before + public void setUp() throws Exception { + originalVcfInputFilePath = inputParameters.getVcf(); + originalVcfContent = getOriginalVcfContent(originalVcfInputFilePath); + writeToTempVCFFile(originalVcfContent, tempVcfInputFileToTestFailingJobs); + + runner.setJobNames(CREATE_SUBSNP_ACCESSION_JOB); + deleteTemporaryContigAndVariantFiles(inputParameters, tempVcfOutputDir); + useOriginalVcfFile(inputParameters, originalVcfInputFilePath, vcfReader); + + mongoTemplate.dropCollection(SubmittedVariantEntity.class); + } + + @Test + @DirtiesContext + public void testContiguousBlocksAreReleasedInCaseOfJobFailures() throws Exception { + useTempVcfFile(inputParameters, tempVcfInputFileToTestFailingJobs, vcfReader); + String modifiedVcfContent = originalVcfContent.replace("76852", "76852jibberish"); + injectErrorIntoTempVcf(modifiedVcfContent, tempVcfInputFileToTestFailingJobs); + // run the job and check the job failed + runner.run(); + assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITH_ERRORS, runner.getExitCode()); + + // Check reserved blocks are being released even in case of job failures + blockRepository.findAll().forEach(block -> { + assertTrue(block.isNotReserved()); + }); + } + +} \ No newline at end of file diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/SSAccessionRecoveryTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/SSAccessionRecoveryTest.java new file mode 100644 index 000000000..16dbca827 --- /dev/null +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/runner/SSAccessionRecoveryTest.java @@ -0,0 +1,227 @@ +package uk.ac.ebi.eva.accession.pipeline.runner; + +/* + * Copyright 2014-2018 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.batch.test.context.SpringBatchTest; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; +import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.entities.ContiguousIdBlock; +import uk.ac.ebi.ampt2d.commons.accession.persistence.jpa.monotonic.repositories.ContiguousIdBlockRepository; +import uk.ac.ebi.eva.accession.core.model.SubmittedVariant; +import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; +import uk.ac.ebi.eva.accession.pipeline.test.SSAccessionRecoveryTestConfiguration; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.SS_ACCESSION_RECOVERY_JOB; + +@RunWith(SpringRunner.class) +@ContextConfiguration(classes = {SSAccessionRecoveryTestConfiguration.class}) +@TestPropertySource("classpath:ss-accession-recovery.properties") +@SpringBatchTest +public class SSAccessionRecoveryTest { + @Autowired + private EvaAccessionJobLauncherCommandLineRunner runner; + + @Autowired + private ContiguousIdBlockRepository blockRepository; + + @Autowired + private SubmittedVariantAccessioningRepository mongoRepository; + + @Test + @DirtiesContext + public void testContiguousBlocksForCategorySSAreRecovered() throws Exception { + initializeMongoDbWithUncommittedAccessions(); + verifyInitialDBState(); + + // recovery cut off time is -14 days (provided in ss-accession-recovery.properties) + runner.setJobNames(SS_ACCESSION_RECOVERY_JOB); + runner.run(); + assertEquals(EvaAccessionJobLauncherCommandLineRunner.EXIT_WITHOUT_ERRORS, runner.getExitCode()); + + verifyEndDBState(); + } + + private void initializeMongoDbWithUncommittedAccessions() { + mongoRepository.deleteAll(); + + List submittedVariantEntityList = new ArrayList<>(); + // Entries for 1st block + for (long i = 5000000000l; i <= 5000000029l; i++) { + SubmittedVariant model = new SubmittedVariant("assembly", 1111, + "project", "contig", 100, "A", "T", + null, false, false, false, + false, null); + SubmittedVariantEntity entity = new SubmittedVariantEntity(i, "hash" + i, model, 1); + submittedVariantEntityList.add(entity); + } + + // Entries for 2nd block - Missing 5 RS (3000000035l - 3000000039l) + for (long i = 5000000030l; i <= 5000000034l; i++) { + SubmittedVariant model = new SubmittedVariant("assembly", 1111, + "project", "contig", 100, "A", "T", + null, false, false, false, + false, null); + SubmittedVariantEntity entity = new SubmittedVariantEntity(i, "hash" + i, model, 1); + submittedVariantEntityList.add(entity); + } + for (long i = 5000000040l; i <= 5000000059l; i++) { + SubmittedVariant model = new SubmittedVariant("assembly", 1111, + "project", "contig", 100, "A", "T", + null, false, false, false, + false, null); + SubmittedVariantEntity entity = new SubmittedVariantEntity(i, "hash" + i, model, 1); + submittedVariantEntityList.add(entity); + } + + // Entries for 3rd block + for (long i = 5000000060l; i <= 5000000089l; i++) { + SubmittedVariant model = new SubmittedVariant("assembly", 1111, + "project", "contig", 100, "A", "T", + null, false, false, false, + false, null); + SubmittedVariantEntity entity = new SubmittedVariantEntity(i, "hash" + i, model, 1); + submittedVariantEntityList.add(entity); + } + + // Entries for 5th block + for (long i = 5000000120l; i <= 5000000129l; i++) { + SubmittedVariant model = new SubmittedVariant("assembly", 1111, + "project", "contig", 100, "A", "T", + null, false, false, false, + false, null); + SubmittedVariantEntity entity = new SubmittedVariantEntity(i, "hash" + i, model, 1); + submittedVariantEntityList.add(entity); + } + + mongoRepository.saveAll(submittedVariantEntityList); + } + + private void verifyInitialDBState() { + // Initial state of Contiguous Id Block DB is 5 blocks are present but their "last_committed" is not updated + // (Initialized using "resources/test-data/ss_accession_recovery_test_data.sql") + + // block id first value last value last committed reserved last_updated_timestamp | remarks + // 1 5000000000 5000000029 4999999999 true 1970-01-01 00:00:00 | should be recovered + // 2 5000000030 5000000059 5000000029 true 1970-01-01 00:00:00 | should be recovered + // 3 5000000060 5000000089 5000000059 true 1970-01-01 00:00:00 | should be recovered + // 4 5000000090 5000000119 5000000089 true 1970-01-01 00:00:00 | should be recovered + // 5 5000000120 5000000149 5000000119 true 2099-01-01 00:00:00 | should not be recovered + + // Mongo DB + // 95 accessions have been used in mongoDB but are not reflected in the block allocation table + // 30 accessions belong to 1st block (5000000000 to 5000000029), + // 25 to the 2nd block (5000000030 to 500000034 and 5000000040 to 5000000059) + // 30 to the 3rd block (5000000060 to 5000000089) + // None in 4th block + // 10 to the 5th block (5000000120 to 5000000129) + + assertEquals(95, mongoRepository.count()); // 30 + 25 + 30 + 10 + assertEquals(5, blockRepository.count()); + + // Since none of the 4 blocks got committed - everyone's last committed value is first_value - 1 + ContiguousIdBlock block1 = blockRepository.findById(1l).get(); + assertEquals(5000000000l, block1.getFirstValue()); + assertEquals(4999999999l, block1.getLastCommitted()); + assertEquals(5000000029l, block1.getLastValue()); + assertEquals("test-instance-recover-state-00", block1.getApplicationInstanceId()); + assertTrue(block1.isReserved()); + + ContiguousIdBlock block2 = blockRepository.findById(2l).get(); + assertEquals(5000000030l, block2.getFirstValue()); + assertEquals(5000000029l, block2.getLastCommitted()); + assertEquals(5000000059l, block2.getLastValue()); + assertEquals("test-instance-recover-state-00", block2.getApplicationInstanceId()); + assertTrue(block2.isReserved()); + + ContiguousIdBlock block3 = blockRepository.findById(3l).get(); + assertEquals(5000000060l, block3.getFirstValue()); + assertEquals(5000000059l, block3.getLastCommitted()); + assertEquals(5000000089l, block3.getLastValue()); + assertEquals("test-instance-recover-state-00", block3.getApplicationInstanceId()); + assertTrue(block3.isReserved()); + + ContiguousIdBlock block4 = blockRepository.findById(4l).get(); + assertEquals(5000000090l, block4.getFirstValue()); + assertEquals(5000000089l, block4.getLastCommitted()); + assertEquals(5000000119l, block4.getLastValue()); + assertEquals("test-instance-recover-state-00", block4.getApplicationInstanceId()); + assertTrue(block4.isReserved()); + + ContiguousIdBlock block5 = blockRepository.findById(5l).get(); + assertEquals(5000000120l, block5.getFirstValue()); + assertEquals(5000000119l, block5.getLastCommitted()); + assertEquals(5000000149l, block5.getLastValue()); + assertEquals("test-instance-recover-state-00", block5.getApplicationInstanceId()); + assertTrue(block5.isReserved()); + } + + private void verifyEndDBState() { + assertEquals(5, blockRepository.count()); + + // Block Recovered - recovered entire block + ContiguousIdBlock block1 = blockRepository.findById(1l).get(); + assertEquals(5000000000l, block1.getFirstValue()); + assertEquals(5000000029l, block1.getLastCommitted()); + assertEquals(5000000029l, block1.getLastValue()); + assertEquals("0", block1.getApplicationInstanceId()); + assertTrue(block1.isNotReserved()); + + // Block Recovered partially - (used 5000000030-5000000034 and 5000000040-5000000059) + // since there are unused accessions(5000000035-5000000039), last_committed is updated to 5000000034 only) + ContiguousIdBlock block2 = blockRepository.findById(2l).get(); + assertEquals(5000000030l, block2.getFirstValue()); + assertEquals(5000000034l, block2.getLastCommitted()); + assertEquals(5000000059l, block2.getLastValue()); + assertEquals("0", block2.getApplicationInstanceId()); + assertTrue(block2.isNotReserved()); + + // Block Recovered - recovered entire block + ContiguousIdBlock block3 = blockRepository.findById(3l).get(); + assertEquals(5000000060l, block3.getFirstValue()); + assertEquals(5000000089l, block3.getLastCommitted()); + assertEquals(5000000089l, block3.getLastValue()); + assertEquals("0", block3.getApplicationInstanceId()); + assertTrue(block3.isNotReserved()); + + // Block Recovered - None of the accessions are used, just released the block + ContiguousIdBlock block4 = blockRepository.findById(4l).get(); + assertEquals(5000000090l, block4.getFirstValue()); + assertEquals(5000000089l, block4.getLastCommitted()); + assertEquals(5000000119l, block4.getLastValue()); + assertEquals("0", block4.getApplicationInstanceId()); + assertTrue(block4.isNotReserved()); + + // Block Not Recovered - Block not to be recovered as it's last update is after cut off time + ContiguousIdBlock block5 = blockRepository.findById(5l).get(); + assertEquals(5000000120l, block5.getFirstValue()); + assertEquals(5000000119l, block5.getLastCommitted()); + assertEquals(5000000149l, block5.getLastValue()); + assertEquals("test-instance-recover-state-00", block5.getApplicationInstanceId()); + assertTrue(block5.isReserved()); + } +} \ No newline at end of file diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/SSAccessionRecoveryTestConfiguration.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/SSAccessionRecoveryTestConfiguration.java new file mode 100644 index 000000000..4e9d3a2b5 --- /dev/null +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/test/SSAccessionRecoveryTestConfiguration.java @@ -0,0 +1,64 @@ +/* + * Copyright 2018 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.ac.ebi.eva.accession.pipeline.test; + +import org.springframework.batch.core.configuration.annotation.BatchConfigurer; +import org.springframework.batch.test.JobLauncherTestUtils; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; +import uk.ac.ebi.eva.accession.pipeline.configuration.batch.io.AccessionWriterConfiguration; +import uk.ac.ebi.eva.accession.pipeline.configuration.batch.jobs.SSAccessionRecoveryJobConfiguration; +import uk.ac.ebi.eva.accession.pipeline.configuration.batch.listeners.SSAccessionRecoveryJobListenerConfiguration; +import uk.ac.ebi.eva.accession.pipeline.configuration.batch.processors.VariantProcessorConfiguration; +import uk.ac.ebi.eva.accession.pipeline.configuration.batch.recovery.SSAccessionRecoveryServiceConfiguration; +import uk.ac.ebi.eva.accession.pipeline.configuration.batch.steps.SSAccessionRecoveryStepConfiguration; +import uk.ac.ebi.eva.accession.pipeline.runner.EvaAccessionJobLauncherCommandLineRunner; +import uk.ac.ebi.eva.commons.batch.configuration.SpringBoot1CompatibilityConfiguration; +import uk.ac.ebi.eva.commons.batch.job.JobExecutionApplicationListener; + +import javax.persistence.EntityManagerFactory; +import javax.sql.DataSource; + +@EnableAutoConfiguration +@Import({SSAccessionRecoveryJobConfiguration.class, + SSAccessionRecoveryStepConfiguration.class, + SSAccessionRecoveryServiceConfiguration.class, + SSAccessionRecoveryJobListenerConfiguration.class, + AccessionWriterConfiguration.class, VariantProcessorConfiguration.class, + EvaAccessionJobLauncherCommandLineRunner.class}) +public class SSAccessionRecoveryTestConfiguration { + @Bean + public BatchConfigurer configurer(DataSource dataSource, EntityManagerFactory entityManagerFactory) + throws Exception { + return SpringBoot1CompatibilityConfiguration.getSpringBoot1CompatibleBatchConfigurer(dataSource, + entityManagerFactory); + } + + @Bean + public JobLauncherTestUtils jobLauncherTestUtils(BatchConfigurer configurer) throws Exception { + JobLauncherTestUtils jobLauncherTestUtils = new JobLauncherTestUtils(); + jobLauncherTestUtils.setJobLauncher(configurer.getJobLauncher()); + jobLauncherTestUtils.setJobRepository(configurer.getJobRepository()); + return jobLauncherTestUtils; + } + + @Bean + public JobExecutionApplicationListener jobExecutionApplicationListener() { + return new JobExecutionApplicationListener(); + } +} \ No newline at end of file diff --git a/eva-accession-pipeline/src/test/resources/properties/ss-accession-recovery.properties b/eva-accession-pipeline/src/test/resources/properties/ss-accession-recovery.properties new file mode 100644 index 000000000..a16c85174 --- /dev/null +++ b/eva-accession-pipeline/src/test/resources/properties/ss-accession-recovery.properties @@ -0,0 +1,46 @@ +recovery.cutoff.days=14 + +spring.datasource.driver-class-name=org.hsqldb.jdbcDriver +spring.datasource.url=jdbc:hsqldb:mem:db;sql.syntax_pgs=true;DB_CLOSE_DELAY=-1 +spring.datasource.username=SA +spring.datasource.password= +spring.datasource.schema=test-data/contiguous_id_blocks_schema.sql +spring.datasource.data=test-data/ss_accession_recovery_test_data.sql +spring.jpa.hibernate.ddl-auto=update + +accessioning.submitted.categoryId=test-pipeline-ss + +eva.count-stats.url=http://localhost:8080 +eva.count-stats.username=username +eva.count-stats.password=password + +accessioning.monotonic.test-pipeline-ss.blockSize=100000 +accessioning.monotonic.test-pipeline-ss.blockStartValue=5000000000 +accessioning.monotonic.test-pipeline-ss.nextBlockInterval=1000000000 + +parameters.assemblyAccession=assembly +parameters.taxonomyAccession=1111 +parameters.projectAccession=project +parameters.chunkSize=5 +parameters.vcf=src/test/resources/input-files/vcf/small_genotyped.vcf.gz +parameters.vcfAggregation=NONE + +parameters.fasta=src/test/resources/input-files/fasta/Homo_sapiens.GRCh37.75.chr20.head_1200.fa +parameters.outputVcf=/tmp/accession-output.vcf +parameters.assemblyReportUrl=file:src/test/resources/input-files/assembly-report/assembly_report.txt +parameters.contigNaming=SEQUENCE_NAME + +spring.jpa.show-sql=true + +spring.data.mongodb.database=test-db +spring.data.mongodb.host=|eva.mongo.host.test| +spring.data.mongodb.password= +spring.data.mongodb.port=27017 +mongodb.read-preference=primary + +# See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding +spring.main.allow-bean-definition-overriding=true + +# to fix exception javax.management.InstanceAlreadyExistsException: com.zaxxer.hikari:name=dataSource,type=HikariDataSource +# see https://stackoverflow.com/a/51798043/2375586 +spring.jmx.enabled=false \ No newline at end of file diff --git a/eva-accession-pipeline/src/test/resources/test-data/ss_accession_recovery_test_data.sql b/eva-accession-pipeline/src/test/resources/test-data/ss_accession_recovery_test_data.sql new file mode 100644 index 000000000..9de052024 --- /dev/null +++ b/eva-accession-pipeline/src/test/resources/test-data/ss_accession_recovery_test_data.sql @@ -0,0 +1,14 @@ +INSERT INTO contiguous_id_blocks +VALUES (1, 'test-instance-recover-state-00', 'ss', 5000000000, 4999999999, 5000000029, true, '1970-01-01 00:00:00'); + +INSERT INTO contiguous_id_blocks +VALUES (2, 'test-instance-recover-state-00', 'ss', 5000000030, 5000000029, 5000000059, true, '1970-01-01 00:00:00'); + +INSERT INTO contiguous_id_blocks +VALUES (3, 'test-instance-recover-state-00', 'ss', 5000000060, 5000000059, 5000000089, true, '1970-01-01 00:00:00'); + +INSERT INTO contiguous_id_blocks +VALUES (4, 'test-instance-recover-state-00', 'ss', 5000000090, 5000000089, 5000000119, true, '1970-01-01 00:00:00'); + +INSERT INTO contiguous_id_blocks +VALUES (5, 'test-instance-recover-state-00', 'ss', 5000000120, 5000000119, 5000000149, true, '2099-01-01 00:00:00'); \ No newline at end of file diff --git a/pom.xml b/pom.xml index 1ed282284..abbf5c55d 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ UTF-8 4.13 0.8.5 - 0.7.14 + 0.7.15