From 1be9e4920be5404268fd230be50d9244ad934892 Mon Sep 17 00:00:00 2001 From: April Shen Date: Wed, 19 Jun 2024 15:25:04 +0100 Subject: [PATCH 1/5] set WriteConcern and ReadPreference for MongoTemplate --- .../core/configuration/nonhuman/MongoConfiguration.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/nonhuman/MongoConfiguration.java b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/nonhuman/MongoConfiguration.java index 52c2bbec1..ad055c95c 100644 --- a/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/nonhuman/MongoConfiguration.java +++ b/eva-accession-core/src/main/java/uk/ac/ebi/eva/accession/core/configuration/nonhuman/MongoConfiguration.java @@ -18,6 +18,8 @@ import com.mongodb.MongoClient; import com.mongodb.MongoClientOptions; +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; @@ -91,6 +93,8 @@ public MongoTemplate mongoTemplate(@Qualifier("primaryFactory") MongoDbFactory m MappingMongoConverter converter) { converter.setTypeMapper(new DefaultMongoTypeMapper(null)); MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory, converter); + mongoTemplate.setWriteConcern(WriteConcern.MAJORITY); + mongoTemplate.setReadPreference(ReadPreference.valueOf(readPreference)); mongoTemplate.setWriteResultChecking(WriteResultChecking.EXCEPTION); return mongoTemplate; } From 80c850c56baeb08748ebea5083814ccae349e22a Mon Sep 17 00:00:00 2001 From: April Shen Date: Thu, 20 Jun 2024 09:26:04 +0100 Subject: [PATCH 2/5] fix count-stats in tests --- ...fSubsnpAccessionsJobConfigurationTest.java | 28 +++++++++++++++++- ...eSubsnpAccessionsJobConfigurationTest.java | 29 ++++++++++++++++++- ...ccession-pipeline-interval-test.properties | 4 +++ 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java index be62853e1..1efdf3cf8 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java @@ -25,11 +25,17 @@ import org.springframework.batch.core.StepExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.client.ExpectedCount; +import org.springframework.test.web.client.MockRestServiceServer; +import org.springframework.web.client.RestTemplate; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; @@ -38,11 +44,13 @@ import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; +import uk.ac.ebi.eva.metrics.count.CountServiceParameters; import java.io.BufferedReader; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; +import java.net.URI; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Arrays; @@ -52,6 +60,9 @@ import java.util.TreeSet; import static org.junit.Assert.assertEquals; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.ACCESSIONING_SHUTDOWN_STEP; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.BUILD_REPORT_STEP; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.CHECK_SUBSNP_ACCESSION_STEP; @@ -76,9 +87,24 @@ public class CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest { @Autowired private MongoTemplate mongoTemplate; + @Autowired + private CountServiceParameters countServiceParameters; + + private final String URL_PATH_SAVE_COUNT = "/v1/bulk/count"; + + @Autowired + @Qualifier("COUNT_STATS_REST_TEMPLATE") + private RestTemplate restTemplate; + + private MockRestServiceServer mockServer; + @Before - public void setUp() { + public void setUp() throws Exception { mongoTemplate.dropCollection(SubmittedVariantEntity.class); + mockServer = MockRestServiceServer.createServer(restTemplate); + mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) + .andExpect(method(HttpMethod.POST)) + .andRespond(withStatus(HttpStatus.OK)); } @After diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java index 3c89c94d9..82a92efad 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java @@ -25,11 +25,17 @@ import org.springframework.batch.core.StepExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.client.ExpectedCount; +import org.springframework.test.web.client.MockRestServiceServer; +import org.springframework.web.client.RestTemplate; import uk.ac.ebi.eva.accession.core.configuration.nonhuman.SubmittedVariantAccessioningConfiguration; import uk.ac.ebi.eva.accession.core.repository.nonhuman.eva.SubmittedVariantAccessioningRepository; @@ -38,12 +44,14 @@ import uk.ac.ebi.eva.accession.pipeline.parameters.InputParameters; import uk.ac.ebi.eva.accession.pipeline.test.BatchTestConfiguration; import uk.ac.ebi.eva.commons.core.utils.FileUtils; +import uk.ac.ebi.eva.metrics.count.CountServiceParameters; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; +import java.net.URI; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Collection; @@ -51,6 +59,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.ACCESSIONING_SHUTDOWN_STEP; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.BUILD_REPORT_STEP; import static uk.ac.ebi.eva.accession.pipeline.configuration.BeanNames.CHECK_SUBSNP_ACCESSION_STEP; @@ -75,9 +86,25 @@ public class CreateSubsnpAccessionsJobConfigurationTest { @Autowired private MongoTemplate mongoTemplate; + + @Autowired + private CountServiceParameters countServiceParameters; + + private final String URL_PATH_SAVE_COUNT = "/v1/bulk/count"; + + @Autowired + @Qualifier("COUNT_STATS_REST_TEMPLATE") + private RestTemplate restTemplate; + + private MockRestServiceServer mockServer; + @Before - public void setUp() { + public void setUp() throws Exception { mongoTemplate.dropCollection(SubmittedVariantEntity.class); + mockServer = MockRestServiceServer.createServer(restTemplate); + mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) + .andExpect(method(HttpMethod.POST)) + .andRespond(withStatus(HttpStatus.OK)); } @After diff --git a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-interval-test.properties b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-interval-test.properties index 69812118f..2c6b400e8 100644 --- a/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-interval-test.properties +++ b/eva-accession-pipeline/src/test/resources/properties/accession-pipeline-interval-test.properties @@ -4,6 +4,10 @@ accessioning.monotonic.test-pipeline-ss.blockSize=2 accessioning.monotonic.test-pipeline-ss.blockStartValue=500 accessioning.monotonic.test-pipeline-ss.nextBlockInterval=10 +eva.count-stats.url=http://localhost:8080 +eva.count-stats.username=username +eva.count-stats.password=password + parameters.assemblyAccession=assembly parameters.taxonomyAccession=1111 parameters.projectAccession=project From 01aec6ab96e5f06bcaf13df6e9a3f7451cc56192 Mon Sep 17 00:00:00 2001 From: April Shen Date: Thu, 20 Jun 2024 10:49:57 +0100 Subject: [PATCH 3/5] clean up --- .../batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java index 82a92efad..12966f77e 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java @@ -86,7 +86,6 @@ public class CreateSubsnpAccessionsJobConfigurationTest { @Autowired private MongoTemplate mongoTemplate; - @Autowired private CountServiceParameters countServiceParameters; From 95dfa93be14deb69f92f3ea334b1ab113d3dbd08 Mon Sep 17 00:00:00 2001 From: April Shen Date: Thu, 20 Jun 2024 13:49:14 +0100 Subject: [PATCH 4/5] add file cleanup to test --- ...alIntervalsOfSubsnpAccessionsJobConfigurationTest.java | 8 ++++++-- .../jobs/CreateSubsnpAccessionsJobConfigurationTest.java | 7 ++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java index 1efdf3cf8..e0c56c850 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest.java @@ -100,6 +100,7 @@ public class CreateSeveralIntervalsOfSubsnpAccessionsJobConfigurationTest { @Before public void setUp() throws Exception { + this.cleanSlate(); mongoTemplate.dropCollection(SubmittedVariantEntity.class); mockServer = MockRestServiceServer.createServer(restTemplate); mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) @@ -109,12 +110,15 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { + this.cleanSlate(); + mongoTemplate.dropCollection(SubmittedVariantEntity.class); + } + + public void cleanSlate() throws Exception { Files.deleteIfExists(Paths.get(inputParameters.getOutputVcf())); Files.deleteIfExists(Paths.get(inputParameters.getOutputVcf() + AccessionReportWriter.VARIANTS_FILE_SUFFIX)); Files.deleteIfExists(Paths.get(inputParameters.getOutputVcf() + AccessionReportWriter.CONTIGS_FILE_SUFFIX)); - Files.deleteIfExists(Paths.get(inputParameters.getOutputVcf())); Files.deleteIfExists(Paths.get(inputParameters.getFasta() + ".fai")); - mongoTemplate.dropCollection(SubmittedVariantEntity.class); } @Test diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java index 12966f77e..0e35e0f44 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/jobs/CreateSubsnpAccessionsJobConfigurationTest.java @@ -99,6 +99,7 @@ public class CreateSubsnpAccessionsJobConfigurationTest { @Before public void setUp() throws Exception { + this.cleanSlate(); mongoTemplate.dropCollection(SubmittedVariantEntity.class); mockServer = MockRestServiceServer.createServer(restTemplate); mockServer.expect(ExpectedCount.manyTimes(), requestTo(new URI(countServiceParameters.getUrl() + URL_PATH_SAVE_COUNT))) @@ -108,11 +109,15 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { + this.cleanSlate(); + mongoTemplate.dropCollection(SubmittedVariantEntity.class); + } + + public void cleanSlate() throws Exception { Files.deleteIfExists(Paths.get(inputParameters.getOutputVcf())); Files.deleteIfExists(Paths.get(inputParameters.getOutputVcf() + AccessionReportWriter.VARIANTS_FILE_SUFFIX)); Files.deleteIfExists(Paths.get(inputParameters.getOutputVcf() + AccessionReportWriter.CONTIGS_FILE_SUFFIX)); Files.deleteIfExists(Paths.get(inputParameters.getFasta() + ".fai")); - mongoTemplate.dropCollection(SubmittedVariantEntity.class); } @Test From 731a06d70c4e5e800596d0f8151ff19e13aeecad Mon Sep 17 00:00:00 2001 From: nkumar2 Date: Thu, 27 Jun 2024 10:33:15 +0100 Subject: [PATCH 5/5] delete files before test start --- .../steps/CreateSubsnpAccessionsStepConfigurationTest.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/CreateSubsnpAccessionsStepConfigurationTest.java b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/CreateSubsnpAccessionsStepConfigurationTest.java index 23a044e6e..f02bdc1c1 100644 --- a/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/CreateSubsnpAccessionsStepConfigurationTest.java +++ b/eva-accession-pipeline/src/test/java/uk/ac/ebi/eva/accession/pipeline/configuration/batch/steps/CreateSubsnpAccessionsStepConfigurationTest.java @@ -77,7 +77,11 @@ public class CreateSubsnpAccessionsStepConfigurationTest { private MongoTemplate mongoTemplate; @Before - public void setUp() { + public void setUp() throws Exception { + Files.deleteIfExists(Paths.get(inputParameters.getOutputVcf())); + Files.deleteIfExists(Paths.get(inputParameters.getOutputVcf() + AccessionReportWriter.VARIANTS_FILE_SUFFIX)); + Files.deleteIfExists(Paths.get(inputParameters.getOutputVcf() + AccessionReportWriter.CONTIGS_FILE_SUFFIX)); + Files.deleteIfExists(Paths.get(inputParameters.getFasta() + ".fai")); mongoTemplate.dropCollection(SubmittedVariantEntity.class); Mockito.when(jobExecution.getJobId()).thenReturn(1L); accessionWriter.setJobExecution(jobExecution);