From 44666f6a1c7c45c045604c167f4120efd19965cd Mon Sep 17 00:00:00 2001 From: Colin Rosenthal Date: Wed, 6 Jan 2016 10:29:33 +0100 Subject: [PATCH] Added test of job splitting for EAV-defined attributes. --- .../netarkivet/systemtest/HarvestUtils.java | 26 +++++++ .../functional/SelectiveHarvestTest.java | 74 +++++++++++++++++-- .../page/DomainConfigurationPageHelper.java | 28 ++++++- .../page/SelectiveHarvestPageHelper.java | 16 ++++ 4 files changed, 138 insertions(+), 6 deletions(-) diff --git a/integration-test/system-test/src/test/java/dk/netarkivet/systemtest/HarvestUtils.java b/integration-test/system-test/src/test/java/dk/netarkivet/systemtest/HarvestUtils.java index abbe2eded0..6c38368ae9 100644 --- a/integration-test/system-test/src/test/java/dk/netarkivet/systemtest/HarvestUtils.java +++ b/integration-test/system-test/src/test/java/dk/netarkivet/systemtest/HarvestUtils.java @@ -135,6 +135,32 @@ public static void waitForJobStatus(String harvestName, JobStatus jobStatus) { } } + public static void waitForJobGeneration(String harvestName) { + boolean keepWaiting = true; + int secondsWaitingForJob = 0; + int maxSecondsToWaitForAllHarvests = 60; + while (keepWaiting) { + System.err.print("."); + + try { + Thread.sleep(10000); + secondsWaitingForJob = secondsWaitingForJob + 10; + } catch (InterruptedException e) { + } + if (secondsWaitingForJob > maxSecondsToWaitForAllHarvests) { + throw new RuntimeException("The job for " + harvestName + " took to long (more that " + + maxSecondsToWaitForAllHarvests + "s) to finish, " + "aborting"); + } + + PageHelper.reloadSubPage("History/Harveststatus-alljobs.jsp?" + + "JOB_STATUS=ALL" + + "&HARVEST_NAME=&START_DATE=&END_DATE=&JOB_ID_ORDER=ASC&PAGE_SIZE=100&START_PAGE_INDEX=1&upload=Show"); + if (PageHelper.getWebDriver().getPageSource().contains(harvestName)) { + keepWaiting = false; + } + } + } + public static String findHarvestingHost() { boolean jobIsRunning = false; int secondsBetweenCheckForRunningJob = 10; diff --git a/integration-test/system-test/src/test/java/dk/netarkivet/systemtest/functional/SelectiveHarvestTest.java b/integration-test/system-test/src/test/java/dk/netarkivet/systemtest/functional/SelectiveHarvestTest.java index 6ff09c1023..774f3733a1 100644 --- a/integration-test/system-test/src/test/java/dk/netarkivet/systemtest/functional/SelectiveHarvestTest.java +++ b/integration-test/system-test/src/test/java/dk/netarkivet/systemtest/functional/SelectiveHarvestTest.java @@ -22,18 +22,31 @@ */ package dk.netarkivet.systemtest.functional; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + import java.lang.reflect.Method; import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Date; +import java.util.List; +import org.apache.commons.lang.RandomStringUtils; import org.openqa.selenium.By; +import org.openqa.selenium.WebElement; +import org.openqa.selenium.support.ui.Select; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import dk.netarkivet.systemtest.AbstractSystemTest; +import dk.netarkivet.systemtest.HarvestUtils; import dk.netarkivet.systemtest.NASAssert; import dk.netarkivet.systemtest.SeleniumTest; +import dk.netarkivet.systemtest.page.DomainConfigurationPageHelper; +import dk.netarkivet.systemtest.page.DomainWebTestHelper; +import dk.netarkivet.systemtest.page.PageHelper; import dk.netarkivet.systemtest.page.SelectiveHarvestPageHelper; /** @@ -61,14 +74,14 @@ public void selectiveHarvestListingTest() throws Exception { addStep("Create a selective harvest", "The harvest should be created successfully a be listed in the HD list"); String harvest1ID = createHarverstID(); SelectiveHarvestPageHelper.createSelectiveHarvest(harvest1ID); - NASAssert.assertTrue(driver.getPageSource().contains(harvest1ID), harvest1ID + assertTrue(driver.getPageSource().contains(harvest1ID), harvest1ID + " not found in harvest list after creation"); addStep("Create a second harvest and active it", "The second harvest also be listed in the HD list"); String harvest2ID = createHarverstID(); SelectiveHarvestPageHelper.createSelectiveHarvest(harvest2ID); SelectiveHarvestPageHelper.activateHarvest(harvest2ID); - NASAssert.assertTrue(driver.getPageSource().contains(harvest2ID), harvest2ID + assertTrue(driver.getPageSource().contains(harvest2ID), harvest2ID + " not found in harvest list after creation"); addStep("Hide inactive harvests", "The harvest first harvest should disappear from the HD list, " @@ -76,19 +89,70 @@ public void selectiveHarvestListingTest() throws Exception { driver.findElement(By.linkText("Hide inactive harvest definitions")).click(); NASAssert.assertFalse(driver.getPageSource().contains(harvest1ID), "Inactive harvest " + harvest1ID + " show in harvest list after 'hide inactive harvests' was clicked"); - NASAssert.assertTrue(driver.getPageSource().contains(harvest2ID), harvest2ID + assertTrue(driver.getPageSource().contains(harvest2ID), harvest2ID + " not found in harvest list after creation"); addStep("Show inactive harvests", "The harvest first harvest should reappear from the HD list, " + "the second should remain"); driver.findElement(By.linkText("Show inactive harvest definitions")).click(); - NASAssert.assertTrue(driver.getPageSource().contains(harvest1ID), "Inactive harvest " + harvest1ID + assertTrue(driver.getPageSource().contains(harvest1ID), "Inactive harvest " + harvest1ID + " show in harvest list after 'hide inactive harvests' was clicked"); - NASAssert.assertTrue(driver.getPageSource().contains(harvest2ID), harvest2ID + assertTrue(driver.getPageSource().contains(harvest2ID), harvest2ID + " not found in harvest list after creation"); } private String createHarverstID() { return harvestIDForTest + "-" + harvestCounter++; } + + /** + * Test creates 8 distinct harvest configurations corresponding to two different values for each of 3 + * parameters defined in the EAV model (2³=8).The eight configurations are placed in a single selective harvest + * and it is checked that eight jobs are created. + */ + @Test(groups = {"guitest", "functest"}) + public void jobSplittingTest() { + final String configName = "newconf"; + List domainList = new ArrayList(); + for (int i=0; i<=7; i++ ) { + domainList.add("d"+i+".dk"); + } + createDomainAndConfiguration(domainList.get(0), configName, 10, false, false); + createDomainAndConfiguration(domainList.get(1), configName, 10, false, true); + createDomainAndConfiguration(domainList.get(2), configName, 10, true, false); + createDomainAndConfiguration(domainList.get(3), configName, 10, true, true); + createDomainAndConfiguration(domainList.get(4), configName, 20, false, false); + createDomainAndConfiguration(domainList.get(5), configName, 20, false, true); + createDomainAndConfiguration(domainList.get(6), configName, 20, true, false); + createDomainAndConfiguration(domainList.get(7), configName, 20, true, true); + final String harvestName = RandomStringUtils.random(6, true, true); + SelectiveHarvestPageHelper.createSelectiveHarvest(harvestName, "", + (String[]) domainList.toArray(new String[] {})); + SelectiveHarvestPageHelper.editHarvest(harvestName); + WebElement table = PageHelper.getWebDriver().findElement(By.className("selection_table")); + List selects = table.findElements(By.tagName("select")); + for (WebElement select: selects) { + Select dropdown = new Select(select); + dropdown.selectByVisibleText(configName); + } + PageHelper.getWebDriver().findElement(By.name("save")).click(); + SelectiveHarvestPageHelper.activateHarvest(harvestName); + HarvestUtils.waitForJobGeneration(harvestName); + List links = PageHelper.getWebDriver().findElements(By.partialLinkText(harvestName)); + assertEquals(links.size(), 8, "Expected to generate one job per distinct configuration."); + } + + + + private static void createDomainAndConfiguration(String domainName, String configurationName, int maxHops, + boolean obeyRobots, boolean extractJavascript) { + DomainWebTestHelper.createDomain(new String[] {domainName}); + DomainConfigurationPageHelper.createConfiguration(domainName, configurationName); + DomainConfigurationPageHelper.gotoConfigurationPage(domainName, configurationName); + DomainConfigurationPageHelper.setMaxHops(maxHops); + DomainConfigurationPageHelper.setHonorRobots(obeyRobots); + DomainConfigurationPageHelper.setExtractJavascript(extractJavascript); + DomainConfigurationPageHelper.submitChanges(); + } + } diff --git a/integration-test/system-test/src/test/java/dk/netarkivet/systemtest/page/DomainConfigurationPageHelper.java b/integration-test/system-test/src/test/java/dk/netarkivet/systemtest/page/DomainConfigurationPageHelper.java index b418045a84..6cdbbc28a1 100644 --- a/integration-test/system-test/src/test/java/dk/netarkivet/systemtest/page/DomainConfigurationPageHelper.java +++ b/integration-test/system-test/src/test/java/dk/netarkivet/systemtest/page/DomainConfigurationPageHelper.java @@ -26,6 +26,7 @@ import org.jaccept.TestEventManager; import org.openqa.selenium.By; +import org.openqa.selenium.Keys; import org.openqa.selenium.WebDriver; import org.openqa.selenium.WebElement; @@ -36,6 +37,9 @@ public class DomainConfigurationPageHelper { public static final String MAX_OBJECTS_FIELD = "maxObjects"; public static final String MAX_BYTES_FIELD = "maxBytes"; public static final String COMMENTS = "comments"; + public static final String MAX_HOPS = "MAX_HOPS"; + public static final String HONOR_ROBOTS_DOT_TXT = "HONOR_ROBOTS_DOT_TXT"; + public static final String EXTRACT_JAVASCRIPT = "EXTRACT_JAVASCRIPT"; public static void createConfiguration(String domainName, String configurationName) { TestEventManager.getInstance().addStimuli("Creating configuration" + configurationName); @@ -57,7 +61,7 @@ public static void gotoDefaultConfigurationPage(String domainName) { public static void gotoConfigurationPage(String domainName, String configurationName) { TestEventManager.getInstance().addStimuli("Updating configuration" + configurationName); DomainWebTestHelper.editDomain(domainName); - + PageHelper.getWebDriver().findElement(By.linkText("Show unused configurations")).click(); WebElement table = PageHelper.getWebDriver().findElement(By.className("selection_table")); List tr_collection = table.findElements(By.tagName("tr")); for (WebElement webElement : tr_collection) { @@ -77,6 +81,28 @@ public static void setMaxObjects(int value) { maxObjectsField.sendKeys(String.valueOf(value)); } + public static void setMaxHops(int value) { + WebElement maxHopsField = PageHelper.getWebDriver().findElement(By.name(MAX_HOPS)); + maxHopsField.clear(); + maxHopsField.sendKeys(String.valueOf(value)); + } + + public static void setHonorRobots(boolean value) { + WebElement honorRobotsBox = PageHelper.getWebDriver().findElement(By.name(HONOR_ROBOTS_DOT_TXT)); + setCheckbox(honorRobotsBox, value); + } + + public static void setExtractJavascript(boolean value) { + WebElement extractJavascriptBox = PageHelper.getWebDriver().findElement(By.name(EXTRACT_JAVASCRIPT)); + setCheckbox(extractJavascriptBox, value); + } + + private static void setCheckbox(WebElement checkbox, boolean value) { + if ( (checkbox.isSelected() && !value) || (!checkbox.isSelected() && value)) { + checkbox.click(); + } + } + public static void submitChanges() { PageHelper.getWebDriver().findElement(By.cssSelector("input[type=\"submit\"]")).click(); } diff --git a/integration-test/system-test/src/test/java/dk/netarkivet/systemtest/page/SelectiveHarvestPageHelper.java b/integration-test/system-test/src/test/java/dk/netarkivet/systemtest/page/SelectiveHarvestPageHelper.java index f7652396fa..bc6e95ebc4 100644 --- a/integration-test/system-test/src/test/java/dk/netarkivet/systemtest/page/SelectiveHarvestPageHelper.java +++ b/integration-test/system-test/src/test/java/dk/netarkivet/systemtest/page/SelectiveHarvestPageHelper.java @@ -83,6 +83,22 @@ public static void activateHarvest(String name) { } } + public static void editHarvest(String name) { + TestEventManager.getInstance().addStimuli("Activating harvest " + name); + PageHelper.gotoPage(PageHelper.MenuPages.SelectiveHarvests); + if (PageHelper.getWebDriver().getPageSource().contains("Show inactive harvest definitions")) { + PageHelper.getWebDriver().findElement(By.linkText("Show inactive harvest definitions")).click(); + } + WebElement table = PageHelper.getWebDriver().findElement(By.className("selection_table")); + List tr_collection = table.findElements(By.tagName("tr")); + for (WebElement webElement : tr_collection) { + if (webElement.getText().contains(name)) { + webElement.findElement(By.linkText("Edit")).click(); + break; + } + } + } + public static void deactivateHarvest(String name) { TestEventManager.getInstance().addStimuli("Deactivating harvest " + name); PageHelper.gotoPage(PageHelper.MenuPages.SelectiveHarvests);