Added test of job splitting for EAV-defined attributes.

integration-test/system-test/src/test/java/dk/netarkivet/systemtest/HarvestUtils.java

@@ -135,6 +135,32 @@ public static void waitForJobStatus(String harvestName, JobStatus jobStatus) {
         }
     }
 
+    public static void waitForJobGeneration(String harvestName) {
+        boolean keepWaiting = true;
+        int secondsWaitingForJob = 0;
+        int maxSecondsToWaitForAllHarvests = 60;
+        while (keepWaiting) {
+            System.err.print(".");
+
+            try {
+                Thread.sleep(10000);
+                secondsWaitingForJob = secondsWaitingForJob + 10;
+            } catch (InterruptedException e) {
+            }
+            if (secondsWaitingForJob > maxSecondsToWaitForAllHarvests) {
+                throw new RuntimeException("The job for " + harvestName + " took to long (more that "
+                        + maxSecondsToWaitForAllHarvests + "s) to finish, " + "aborting");
+            }
+
+            PageHelper.reloadSubPage("History/Harveststatus-alljobs.jsp?" +
+                    "JOB_STATUS=ALL" +
+                    "&HARVEST_NAME=&START_DATE=&END_DATE=&JOB_ID_ORDER=ASC&PAGE_SIZE=100&START_PAGE_INDEX=1&upload=Show");
+            if (PageHelper.getWebDriver().getPageSource().contains(harvestName)) {
+                keepWaiting = false;
+            }
+        }
+    }
+
     public static String findHarvestingHost() {
         boolean jobIsRunning = false;
         int secondsBetweenCheckForRunningJob = 10;

integration-test/system-test/src/test/java/dk/netarkivet/systemtest/functional/SelectiveHarvestTest.java

@@ -22,18 +22,31 @@
  */
 package dk.netarkivet.systemtest.functional;
 
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
 import java.lang.reflect.Method;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
 import java.util.Date;
+import java.util.List;
 
+import org.apache.commons.lang.RandomStringUtils;
 import org.openqa.selenium.By;
+import org.openqa.selenium.WebElement;
+import org.openqa.selenium.support.ui.Select;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
 import dk.netarkivet.systemtest.AbstractSystemTest;
+import dk.netarkivet.systemtest.HarvestUtils;
 import dk.netarkivet.systemtest.NASAssert;
 import dk.netarkivet.systemtest.SeleniumTest;
+import dk.netarkivet.systemtest.page.DomainConfigurationPageHelper;
+import dk.netarkivet.systemtest.page.DomainWebTestHelper;
+import dk.netarkivet.systemtest.page.PageHelper;
 import dk.netarkivet.systemtest.page.SelectiveHarvestPageHelper;
 
 /**
@@ -61,34 +74,85 @@ public void selectiveHarvestListingTest() throws Exception {
         addStep("Create a selective harvest", "The harvest should be created successfully a be listed in the HD list");
         String harvest1ID = createHarverstID();
         SelectiveHarvestPageHelper.createSelectiveHarvest(harvest1ID);
-        NASAssert.assertTrue(driver.getPageSource().contains(harvest1ID), harvest1ID
+        assertTrue(driver.getPageSource().contains(harvest1ID), harvest1ID
                 + " not found in harvest list after creation");
 
         addStep("Create a second harvest and active it", "The second harvest also be listed in the HD list");
         String harvest2ID = createHarverstID();
         SelectiveHarvestPageHelper.createSelectiveHarvest(harvest2ID);
         SelectiveHarvestPageHelper.activateHarvest(harvest2ID);
-        NASAssert.assertTrue(driver.getPageSource().contains(harvest2ID), harvest2ID
+        assertTrue(driver.getPageSource().contains(harvest2ID), harvest2ID
                 + " not found in harvest list after creation");
 
         addStep("Hide inactive harvests", "The harvest first harvest should disappear from the HD list, "
                 + "the second should remain");
         driver.findElement(By.linkText("Hide inactive harvest definitions")).click();
         NASAssert.assertFalse(driver.getPageSource().contains(harvest1ID), "Inactive harvest " + harvest1ID
                 + " show in harvest list after 'hide inactive harvests' was clicked");
-        NASAssert.assertTrue(driver.getPageSource().contains(harvest2ID), harvest2ID
+        assertTrue(driver.getPageSource().contains(harvest2ID), harvest2ID
                 + " not found in harvest list after creation");
 
         addStep("Show inactive harvests", "The harvest first harvest should reappear from the HD list, "
                 + "the second should remain");
         driver.findElement(By.linkText("Show inactive harvest definitions")).click();
-        NASAssert.assertTrue(driver.getPageSource().contains(harvest1ID), "Inactive harvest " + harvest1ID
+        assertTrue(driver.getPageSource().contains(harvest1ID), "Inactive harvest " + harvest1ID
                 + " show in harvest list after 'hide inactive harvests' was clicked");
-        NASAssert.assertTrue(driver.getPageSource().contains(harvest2ID), harvest2ID
+        assertTrue(driver.getPageSource().contains(harvest2ID), harvest2ID
                 + " not found in harvest list after creation");
     }
 
     private String createHarverstID() {
         return harvestIDForTest + "-" + harvestCounter++;
     }
+
+    /**
+     * Test creates 8 distinct harvest configurations corresponding to two different values for each of 3
+     * parameters defined in the EAV model (2³=8).The eight configurations are placed in a single selective harvest
+     * and it is checked that eight jobs are created.
+     */
+    @Test(groups = {"guitest", "functest"})
+    public void jobSplittingTest() {
+        final String configName = "newconf";
+        List<String> domainList = new ArrayList<String>();
+        for (int i=0; i<=7; i++ ) {
+            domainList.add("d"+i+".dk");
+        }
+        createDomainAndConfiguration(domainList.get(0), configName, 10, false, false);
+        createDomainAndConfiguration(domainList.get(1), configName, 10, false, true);
+        createDomainAndConfiguration(domainList.get(2), configName, 10, true, false);
+        createDomainAndConfiguration(domainList.get(3), configName, 10, true, true);
+        createDomainAndConfiguration(domainList.get(4), configName, 20, false, false);
+        createDomainAndConfiguration(domainList.get(5), configName, 20, false, true);
+        createDomainAndConfiguration(domainList.get(6), configName, 20, true, false);
+        createDomainAndConfiguration(domainList.get(7), configName, 20, true, true);
+        final String harvestName = RandomStringUtils.random(6, true, true);
+        SelectiveHarvestPageHelper.createSelectiveHarvest(harvestName, "",
+                (String[]) domainList.toArray(new String[] {}));
+        SelectiveHarvestPageHelper.editHarvest(harvestName);
+        WebElement table = PageHelper.getWebDriver().findElement(By.className("selection_table"));
+        List<WebElement> selects = table.findElements(By.tagName("select"));
+        for (WebElement select: selects) {
+            Select dropdown = new Select(select);
+            dropdown.selectByVisibleText(configName);
+        }
+        PageHelper.getWebDriver().findElement(By.name("save")).click();
+        SelectiveHarvestPageHelper.activateHarvest(harvestName);
+        HarvestUtils.waitForJobGeneration(harvestName);
+        List<WebElement> links = PageHelper.getWebDriver().findElements(By.partialLinkText(harvestName));
+        assertEquals(links.size(), 8, "Expected to generate one job per distinct configuration.");
+    }
+
+
+
+    private static void createDomainAndConfiguration(String domainName, String configurationName, int maxHops,
+            boolean obeyRobots, boolean extractJavascript) {
+        DomainWebTestHelper.createDomain(new String[] {domainName});
+        DomainConfigurationPageHelper.createConfiguration(domainName, configurationName);
+        DomainConfigurationPageHelper.gotoConfigurationPage(domainName, configurationName);
+        DomainConfigurationPageHelper.setMaxHops(maxHops);
+        DomainConfigurationPageHelper.setHonorRobots(obeyRobots);
+        DomainConfigurationPageHelper.setExtractJavascript(extractJavascript);
+        DomainConfigurationPageHelper.submitChanges();
+    }
+
 }

integration-test/system-test/src/test/java/dk/netarkivet/systemtest/page/DomainConfigurationPageHelper.java

@@ -26,6 +26,7 @@
 
 import org.jaccept.TestEventManager;
 import org.openqa.selenium.By;
+import org.openqa.selenium.Keys;
 import org.openqa.selenium.WebDriver;
 import org.openqa.selenium.WebElement;
 
@@ -36,6 +37,9 @@ public class DomainConfigurationPageHelper {
     public static final String MAX_OBJECTS_FIELD = "maxObjects";
     public static final String MAX_BYTES_FIELD = "maxBytes";
     public static final String COMMENTS = "comments";
+    public static final String MAX_HOPS = "MAX_HOPS";
+    public static final String HONOR_ROBOTS_DOT_TXT = "HONOR_ROBOTS_DOT_TXT";
+    public static final String EXTRACT_JAVASCRIPT = "EXTRACT_JAVASCRIPT";
 
     public static void createConfiguration(String domainName, String configurationName) {
         TestEventManager.getInstance().addStimuli("Creating configuration" + configurationName);
@@ -57,7 +61,7 @@ public static void gotoDefaultConfigurationPage(String domainName) {
     public static void gotoConfigurationPage(String domainName, String configurationName) {
         TestEventManager.getInstance().addStimuli("Updating configuration" + configurationName);
         DomainWebTestHelper.editDomain(domainName);
-
+        PageHelper.getWebDriver().findElement(By.linkText("Show unused configurations")).click();
         WebElement table = PageHelper.getWebDriver().findElement(By.className("selection_table"));
         List<WebElement> tr_collection = table.findElements(By.tagName("tr"));
         for (WebElement webElement : tr_collection) {
@@ -77,6 +81,28 @@ public static void setMaxObjects(int value) {
         maxObjectsField.sendKeys(String.valueOf(value));
     }
 
+    public static void setMaxHops(int value) {
+        WebElement maxHopsField = PageHelper.getWebDriver().findElement(By.name(MAX_HOPS));
+        maxHopsField.clear();
+        maxHopsField.sendKeys(String.valueOf(value));
+    }
+
+    public static void setHonorRobots(boolean value) {
+        WebElement honorRobotsBox = PageHelper.getWebDriver().findElement(By.name(HONOR_ROBOTS_DOT_TXT));
+        setCheckbox(honorRobotsBox, value);
+    }
+
+    public static void setExtractJavascript(boolean value) {
+        WebElement extractJavascriptBox = PageHelper.getWebDriver().findElement(By.name(EXTRACT_JAVASCRIPT));
+        setCheckbox(extractJavascriptBox, value);
+    }
+
+    private static void setCheckbox(WebElement checkbox, boolean value) {
+        if ( (checkbox.isSelected() && !value) || (!checkbox.isSelected() && value)) {
+            checkbox.click();
+        }
+    }
+
     public static void submitChanges() {
         PageHelper.getWebDriver().findElement(By.cssSelector("input[type=\"submit\"]")).click();
     }

integration-test/system-test/src/test/java/dk/netarkivet/systemtest/page/SelectiveHarvestPageHelper.java

@@ -83,6 +83,22 @@ public static void activateHarvest(String name) {
         }
     }
 
+    public static void editHarvest(String name) {
+        TestEventManager.getInstance().addStimuli("Activating harvest " + name);
+        PageHelper.gotoPage(PageHelper.MenuPages.SelectiveHarvests);
+        if (PageHelper.getWebDriver().getPageSource().contains("Show inactive harvest definitions")) {
+            PageHelper.getWebDriver().findElement(By.linkText("Show inactive harvest definitions")).click();
+        }
+        WebElement table = PageHelper.getWebDriver().findElement(By.className("selection_table"));
+        List<WebElement> tr_collection = table.findElements(By.tagName("tr"));
+        for (WebElement webElement : tr_collection) {
+            if (webElement.getText().contains(name)) {
+                webElement.findElement(By.linkText("Edit")).click();
+                break;
+            }
+        }
+    }
+
     public static void deactivateHarvest(String name) {
         TestEventManager.getInstance().addStimuli("Deactivating harvest " + name);
         PageHelper.gotoPage(PageHelper.MenuPages.SelectiveHarvests);