From 65c50690fed5740c4568ae50b3803a9c9e94c3fc Mon Sep 17 00:00:00 2001 From: Matthew Gerring Date: Wed, 22 Feb 2017 15:10:19 +0000 Subject: [PATCH 1/2] Improved duplicate handling in filter mechanism. Decided to support duplicates. --- .../eclipse/scanning/api/filter/Filter.java | 15 ++- .../scanning/test/filter/FilterTest.java | 119 +++++++++++++++++- 2 files changed, 127 insertions(+), 7 deletions(-) diff --git a/org.eclipse.scanning.api/src/org/eclipse/scanning/api/filter/Filter.java b/org.eclipse.scanning.api/src/org/eclipse/scanning/api/filter/Filter.java index a7cc1a03d..0983f145b 100644 --- a/org.eclipse.scanning.api/src/org/eclipse/scanning/api/filter/Filter.java +++ b/org.eclipse.scanning.api/src/org/eclipse/scanning/api/filter/Filter.java @@ -13,6 +13,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; import java.util.List; /** @@ -138,7 +139,19 @@ public List filter(final Collection items) { Collection excludes = match(getExcludes(), items); ret.removeAll(excludes); Collection includes = match(getIncludes(), items); - ret.addAll(includes); + + Collection done = new HashSet<>(); + for (final String item : includes) { + + if (done.contains(item)) continue; + int ecount = (int)ret.stream().filter(t->t.equals(item)).count(); + int icount = (int)includes.stream().filter(t->t.equals(item)).count(); + if (icount>ecount) { + for (int i = 0; i < (icount-ecount); i++) ret.add(item); + } + + done.add(item); + } return ret; } diff --git a/org.eclipse.scanning.test/src/org/eclipse/scanning/test/filter/FilterTest.java b/org.eclipse.scanning.test/src/org/eclipse/scanning/test/filter/FilterTest.java index 14a348d6c..1eb49c000 100644 --- a/org.eclipse.scanning.test/src/org/eclipse/scanning/test/filter/FilterTest.java +++ b/org.eclipse.scanning.test/src/org/eclipse/scanning/test/filter/FilterTest.java @@ -16,8 +16,10 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; import org.eclipse.scanning.api.device.IScannableDeviceService; import org.eclipse.scanning.api.filter.Filter; @@ -26,9 +28,13 @@ import org.eclipse.scanning.api.scan.ScanningException; import org.eclipse.scanning.example.scannable.MockScannableConnector; import org.eclipse.scanning.server.application.PseudoSpringParser; +import org.junit.Before; import org.junit.BeforeClass; +import org.junit.FixMethodOrder; import org.junit.Test; +import org.junit.runners.MethodSorters; +@FixMethodOrder(MethodSorters.NAME_ASCENDING) public class FilterTest { private static IScannableDeviceService sservice; @@ -41,6 +47,13 @@ public static void parseSpring() throws Exception { sservice = new MockScannableConnector(null); } + @Before + public void before() throws Exception { + fservice.clear(); + PseudoSpringParser parser = new PseudoSpringParser(); + parser.parse(FilterTest.class.getResourceAsStream("test_filters.xml")); + } + @Test public void notNull() { assertNotNull(IFilterService.DEFAULT); @@ -50,17 +63,12 @@ public void notNull() { } @Test - public void noFilter() throws ScanningException { + public void znoFilter() throws ScanningException { assertEquals(sservice.getScannableNames(), fservice.filter("not.there", sservice.getScannableNames())); } @Test public void testFilterSpring() throws Exception { - - fservice.clear(); - PseudoSpringParser parser = new PseudoSpringParser(); - parser.parse(FilterTest.class.getResourceAsStream("test_filters.xml")); - check(); } @@ -78,6 +86,105 @@ public void testFilterManual() throws Exception { check(); } + + @Test + public void testDuplicatesNoSpring() throws Exception { + + fservice.clear(); + + IFilter dfilter = new Filter(); + dfilter.setName("duplicates"); + dfilter.setIncludes(Arrays.asList("a")); + dfilter.setExcludes(Arrays.asList("b")); + fservice.register(dfilter); + + List items = new ArrayList<>(Arrays.asList("a", "a", "b", "b")); + assertEquals(Arrays.asList("a", "a"), fservice.filter("duplicates", items)); + + items = new ArrayList<>(Arrays.asList("a", "b", "a", "b")); + assertEquals(Arrays.asList("a", "a"), fservice.filter("duplicates", items)); + + items = new ArrayList<>(Arrays.asList("b", "b", "1", "a", "a")); + assertEquals(Arrays.asList("1", "a", "a"), fservice.filter("duplicates", items)); + + items = new ArrayList<>(Arrays.asList("b", "b", "a", "a", "1")); + assertEquals(Arrays.asList("a", "a", "1"), fservice.filter("duplicates", items)); + + } + + @Test + public void testDuplicatesExclude() throws Exception { + + fservice.clear(); + + IFilter dfilter = new Filter(); + dfilter.setName("duplicates"); + dfilter.setIncludes(Arrays.asList("a")); + dfilter.setExcludes(Arrays.asList("a", "b")); + fservice.register(dfilter); + + List items = new ArrayList<>(Arrays.asList("a", "a", "b", "b")); + assertEquals(Arrays.asList("a", "a"), fservice.filter("duplicates", items)); + + items = new ArrayList<>(Arrays.asList("a", "b", "a", "b")); + assertEquals(Arrays.asList("a", "a"), fservice.filter("duplicates", items)); + + items = new ArrayList<>(Arrays.asList("b", "b", "1", "a", "a")); + assertEquals(Arrays.asList("1", "a", "a"), fservice.filter("duplicates", items)); + + items = new ArrayList<>(Arrays.asList("b", "b", "a", "a", "1")); + + // Because we excluded it the result changes + assertEquals(Arrays.asList("1", "a", "a"), fservice.filter("duplicates", items)); + + } + + @Test(timeout=10000) // Must complete in 10s or less + public void testDuplicatesLarge() throws Exception { + + fservice.clear(); + + IFilter dfilter = new Filter(); + dfilter.setName("duplicates"); + + List as = Arrays.stream(new String[1000]).map(nothing->"a").collect(Collectors.toList()); + assertEquals(1000, as.size()); + dfilter.setIncludes(as); + List bs = Arrays.stream(new String[1000]).map(nothing->"b").collect(Collectors.toList()); + assertEquals(1000, bs.size()); + dfilter.setExcludes(bs); + fservice.register(dfilter); + + List items = new ArrayList<>(); + items.addAll(as); + items.addAll(bs); + assertEquals(as, fservice.filter("duplicates", items)); + + items = new ArrayList<>(); + for (int i = 0; i < 1000; i++) { + items.add(as.get(i)); + items.add(bs.get(i)); + } + assertEquals(as, fservice.filter("duplicates", items)); + + items = new ArrayList<>(); + items.addAll(bs); + items.add("1"); + items.addAll(as); + assertEquals("1", fservice.filter("duplicates", items).get(0)); + assertEquals("a", fservice.filter("duplicates", items).get(1)); + assertEquals("a", fservice.filter("duplicates", items).get(2)); + + items = new ArrayList<>(); + items.addAll(bs); + items.addAll(as); + items.add("1"); + List filtered = fservice.filter("duplicates", items); + assertEquals("1", filtered.get(filtered.size()-1)); + assertEquals("a", filtered.get(filtered.size()-2)); + assertEquals("a", filtered.get(filtered.size()-3)); + + } private void check() throws ScanningException { From 81ec79ada0f2d7a2ded5024b37798a0b5ddb007e Mon Sep 17 00:00:00 2001 From: Matthew Gerring Date: Wed, 22 Feb 2017 15:14:55 +0000 Subject: [PATCH 2/2] Used another lambda, made a little faster. --- .../src/org/eclipse/scanning/api/filter/Filter.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/org.eclipse.scanning.api/src/org/eclipse/scanning/api/filter/Filter.java b/org.eclipse.scanning.api/src/org/eclipse/scanning/api/filter/Filter.java index 0983f145b..fb6df74e5 100644 --- a/org.eclipse.scanning.api/src/org/eclipse/scanning/api/filter/Filter.java +++ b/org.eclipse.scanning.api/src/org/eclipse/scanning/api/filter/Filter.java @@ -12,9 +12,11 @@ package org.eclipse.scanning.api.filter; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.List; +import java.util.stream.Collectors; /** * A springable class which encapsulates information for a given feature. @@ -146,9 +148,7 @@ public List filter(final Collection items) { if (done.contains(item)) continue; int ecount = (int)ret.stream().filter(t->t.equals(item)).count(); int icount = (int)includes.stream().filter(t->t.equals(item)).count(); - if (icount>ecount) { - for (int i = 0; i < (icount-ecount); i++) ret.add(item); - } + ret.addAll(Arrays.stream(new String[icount-ecount]).map(nothing->item).collect(Collectors.toList())); done.add(item); }