Skip to content
Permalink
Browse files

Fixed new comparator and added a unit test for sorting.

  • Loading branch information...
csrster committed Feb 15, 2016
1 parent 8e5f145 commit d8e3dd42e09f51fa90a47a54ad3843b053794937
@@ -146,45 +146,82 @@ public AttributeAndType(AttributeTypeBase attributeType, AttributeBase attribute
}

public static int compare2 (List<AttributeAndType> antList1, List<AttributeAndType> antList2) {
if (antList1 == null) {
antList1 = new ArrayList<>();
}
if (antList2 == null) {
antList2 = new ArrayList<>();
}
int size1 = antList1.size();
int size2 = antList2.size();
if (size1 != size2) {
return (size1 - size2) < 0 ? -1 : 1;
if (!antList1.isEmpty() && !antList2.isEmpty() && size1 != size2) {
//return (size1 - size2) < 0 ? -1 : 1;
throw new UnsupportedOperationException("Haven't though about how to compare attribute lists of different lengths");
}
if (antList1.isEmpty() && antList2.isEmpty()) {
return 0;
}
Collections.sort(antList1);
Collections.sort(antList2);
for (int i=0; i < size1; i++) {
AttributeAndType ant1 = antList1.get(i);
AttributeAndType ant2 = antList2.get(i);
if (ant1.attributeType.id != ant2.attributeType.id) {
return (ant1.attributeType.id - ant2.attributeType.id) < 0 ? -1 : 1;
}
if (ant1.attributeType.datatype != 1) {
throw new UnsupportedOperationException("EAV attribute datatype compare not implemented yet.");
}
Integer i1 = null;
Integer i2 = null;
if (ant1.attribute != null) {
i1 = ant1.attribute.getInteger();
}
if (i1 == null) {
i1 = ant1.attributeType.def_int;
}
if (i1 == null) {
i1 = 0;
}
if (ant2.attribute != null) {
i2 = ant2.attribute.getInteger();
}
if (i2 == null) {
i2 = ant2.attributeType.def_int;
}
if (i2 == null) {
i2 = 0;
if (antList1.size() == antList2.size()) {
for (int i = 0; i < size1; i++) {
AttributeAndType ant1 = antList1.get(i);
AttributeAndType ant2 = antList2.get(i);
if (ant1.attributeType.id != ant2.attributeType.id) {
return (ant1.attributeType.id - ant2.attributeType.id) < 0 ? -1 : 1;
}
if (ant1.attributeType.datatype != 1) {
throw new UnsupportedOperationException("EAV attribute datatype compare not implemented yet.");
}
Integer i1 = null;
Integer i2 = null;
if (ant1.attribute != null) {
i1 = ant1.attribute.getInteger();
}
if (i1 == null) {
i1 = ant1.attributeType.def_int;
}
if (i1 == null) {
i1 = 0;
}
if (ant2.attribute != null) {
i2 = ant2.attribute.getInteger();
}
if (i2 == null) {
i2 = ant2.attributeType.def_int;
}
if (i2 == null) {
i2 = 0;
}
int res = i2 - i1;
if (res != 0L) {
return res < 0L ? -1 : 1;
}
}
int res = i2 - i1;
if (res != 0L) {
return res < 0L ? -1 : 1;
} else {
for (int i = 0; i < Math.max(size1, size2); i++) {
Integer i1 = 0;
Integer i2 = 0;
if (!antList1.isEmpty()) {
AttributeAndType ant1 = antList1.get(i);
i1 = ant1.attribute.getInteger();
i2 = ant1.attributeType.def_int;
if (i1 == null) {
i1 = i2;
}
}
if (!antList2.isEmpty()) {
AttributeAndType ant2 = antList2.get(i);
i2 = ant2.attribute.getInteger();
i1 = ant2.attributeType.def_int;
if (i2 == null) {
i2 = i1;
}
}
int res = i2 - i1;
if (res != 0L) {
return res < 0L ? -1 : 1;
}
}
}
return 0;
@@ -33,11 +33,15 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.GregorianCalendar;
import java.util.List;

import org.junit.Test;

import dk.netarkivet.common.exceptions.PermissionDenied;
import dk.netarkivet.common.exceptions.UnknownID;
import dk.netarkivet.harvester.datamodel.eav.ContentAttrType_Generic;
import dk.netarkivet.harvester.datamodel.eav.ContentAttribute_Generic;
import dk.netarkivet.harvester.datamodel.eav.EAV;
import dk.netarkivet.harvester.test.utils.OrderXmlBuilder;

/**
@@ -170,7 +174,7 @@ public void testExpectedNumberOfObjects() {
public void testExpectedNumberOfObjectsNewerSmallUnfinishedHarvest() {
DomainConfiguration dc = createDefaultDomainConfiguration();
addHistoryObject( dc, 200L, 1L, StopReason.DOWNLOAD_COMPLETE);
addSecondHistoryObject( dc, 100L, 1L, StopReason.SIZE_LIMIT);
addSecondHistoryObject(dc, 100L, 1L, StopReason.SIZE_LIMIT);
assertEquals("Newer smaller unfinished harvest should not affect expectation", 400L,
dc.getExpectedNumberOfObjects(2200L, -1L));
}
@@ -234,7 +238,7 @@ public void testSetSeedlist() {
Domain mockDomain = mock(Domain.class);
when(mockDomain.getSeedList(seedlistName)).thenReturn(domainSeedList);

dc.setSeedLists(mockDomain, Arrays.asList (new SeedList[] {domainSeedList}));
dc.setSeedLists(mockDomain, Arrays.asList(new SeedList[] {domainSeedList}));
SeedList confSeedList = dc.getSeedLists().next();
assertEquals(domainSeedList, confSeedList);
}
@@ -262,6 +266,8 @@ public void testSetSeedlistInconsistententSeedList() {
dc.setSeedLists(mockDomain, Arrays.asList (new SeedList[] {newSeedList}));
}



private void addHistoryObject(DomainConfiguration dc, final long countObjectRetrieved,
long sizeDataRetrieved, final StopReason stopReason) {
dc.getDomainhistory().addHarvestInfo(
@@ -290,4 +296,51 @@ public static DomainConfiguration createDefaultDomainConfiguration(String name)
domainConfiguration.setOrderXmlName(OrderXmlBuilder.DEFAULT_ORDER_XML_NAME);
return domainConfiguration;
}



public static List<EAV.AttributeAndType> getAttributes(int maxHops, boolean obeyRobots, boolean extractJS) {
List<EAV.AttributeAndType> attributeAndTypes = new ArrayList<>();
ContentAttrType_Generic atMaxHops = new ContentAttrType_Generic();
atMaxHops.tree_id = 2;
atMaxHops.id = 1;
atMaxHops.datatype = 1;
atMaxHops.viewtype = 1;
atMaxHops.def_int = 20;
ContentAttribute_Generic aatMaxHops = new ContentAttribute_Generic(atMaxHops);
aatMaxHops.setInteger(maxHops);
attributeAndTypes.add(new EAV.AttributeAndType(atMaxHops, aatMaxHops));

ContentAttrType_Generic atHonorRobots = new ContentAttrType_Generic();
atHonorRobots.tree_id = 2;
atHonorRobots.id = 2;
atHonorRobots.datatype = 1;
atHonorRobots.viewtype = 6;
atHonorRobots.def_int = 0;
ContentAttribute_Generic aatHonorRobots = new ContentAttribute_Generic(atHonorRobots);
if (obeyRobots) {
aatHonorRobots.setInteger(1);
} else {
aatHonorRobots.setInteger(0);
}
attributeAndTypes.add(new EAV.AttributeAndType(atHonorRobots, aatHonorRobots));

ContentAttrType_Generic atExtractJS = new ContentAttrType_Generic();
atExtractJS.tree_id = 2;
atExtractJS.id = 3;
atExtractJS.datatype = 1;
atExtractJS.viewtype = 5;
atExtractJS.def_int = 1;
ContentAttribute_Generic aatExtractJS = new ContentAttribute_Generic(atExtractJS);
if (extractJS) {
aatExtractJS.setInteger(1);
} else {
aatExtractJS.setInteger(0);
}
attributeAndTypes.add(new EAV.AttributeAndType(atExtractJS, aatExtractJS));

return attributeAndTypes;
}


}
@@ -1,8 +1,14 @@
package dk.netarkivet.harvester.scheduler.jobgen;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;

import org.junit.Test;

import dk.netarkivet.common.exceptions.ArgumentNotValid;
@@ -15,6 +21,7 @@
import dk.netarkivet.harvester.datamodel.Job;
import dk.netarkivet.harvester.datamodel.JobTest;
import dk.netarkivet.harvester.datamodel.TestInfo;
import dk.netarkivet.harvester.datamodel.eav.EAV;
import dk.netarkivet.harvester.test.utils.OrderXmlBuilder;

public class DefaultJobGeneratorTest extends AbstractJobGeneratorTest {
@@ -85,4 +92,50 @@ public void testCanAcceptNullConfiguration() {
Job job = JobTest.createDefaultJob();
createJobGenerator().canAccept(job, null, null);
}

@Test
public void testDefaultAttributes() {
DomainConfiguration dc1 = DomainConfigurationTest.createDefaultDomainConfiguration("1.dk");
DomainConfiguration dc2 = DomainConfigurationTest.createDefaultDomainConfiguration("2.dk");
dc1.setAttributesAndTypes(DomainConfigurationTest.getAttributes(20, false, true));
dc2.setAttributesAndTypes(new ArrayList<EAV.AttributeAndType>());
assertEquals(0, EAV.compare2(dc1.getAttributesAndTypes(), dc2.getAttributesAndTypes()));
}

@Test
public void testSorting() {
List<DomainConfiguration> dcs = new ArrayList<>();
dcs.add(getDomainConfiguration("1.dk", 4000000L, 10, false, true));
dcs.add(getDomainConfiguration("2.dk", 2000000L, 20, false, true));
dcs.add(getDomainConfiguration("3.dk", 4000000L, 10, false, true));
dcs.add(getDomainConfiguration("4.dk", 2000000L, 20, false, false));
dcs.add(getDomainConfiguration("5.dk", 2000000L, 20, false, true));
DomainConfiguration dc6 = DomainConfigurationTest.createDefaultDomainConfiguration("6.dk");
dc6.setMaxBytes(2000000L);
dcs.add(dc6); //default domain, should get values (20, false, true)
Comparator<DomainConfiguration> comparator = new DefaultJobGenerator.CompareConfigsDesc(-1, 10000000L);
Collections.sort(dcs, comparator);
List<String> sortedNames = new ArrayList<>();
for (DomainConfiguration dc:dcs) {
sortedNames.add(dc.getDomainName());
}
List<String> expected1 = new ArrayList<>();
expected1.add("2.dk");
expected1.add("5.dk");
expected1.add("6.dk");
assertFalse(Collections.indexOfSubList(sortedNames, expected1) == -1);
List<String> expected2 = new ArrayList<>();
expected2.add("1.dk");
expected2.add("3.dk");
assertFalse(Collections.indexOfSubList(sortedNames, expected2) == -1);
}

public static DomainConfiguration getDomainConfiguration(String name, Long maxBytes, int maxHops,
boolean obeyRobots, boolean extractJS) {
DomainConfiguration dc1 = DomainConfigurationTest.createDefaultDomainConfiguration(name);
dc1.setMaxBytes(maxBytes);
dc1.setAttributesAndTypes(DomainConfigurationTest.getAttributes(maxHops, obeyRobots, extractJS));
return dc1;
}

}

0 comments on commit d8e3dd4

Please sign in to comment.
You can’t perform that action at this time.