From 799fc1a9ed7c5883036146e0d5f96426e6ee7dd2 Mon Sep 17 00:00:00 2001 From: William Burns Date: Thu, 10 Mar 2016 16:48:21 -0500 Subject: [PATCH] ISPN-6354 Remove Map/Reduce --- .../org/infinispan/cdi/ContextInputCache.java | 9 +- .../cdi/DelegatingMapReduceTaskLifecycle.java | 125 -- .../main/java/org/infinispan/cdi/Input.java | 3 +- ...texec.mapreduce.spi.MapReduceTaskLifecycle | 1 - .../distexec/WordCountMapReduceCDITest.java | 128 -- .../infinispan/client/hotrod/ExecTest.java | 16 - .../infinispan/commands/CommandsFactory.java | 26 - .../commands/CommandsFactoryImpl.java | 30 +- .../commands/CreateCacheCommand.java | 27 +- .../commands/RemoteCommandsFactory.java | 8 - .../commands/read/MapCombineCommand.java | 230 --- .../commands/read/ReduceCommand.java | 184 --- .../distexec/mapreduce/Collator.java | 27 - .../distexec/mapreduce/Collector.java | 24 - .../mapreduce/MapReduceCacheLoaderTask.java | 37 - .../mapreduce/MapReduceException.java | 26 - .../distexec/mapreduce/MapReduceManager.java | 96 -- .../mapreduce/MapReduceManagerFactory.java | 24 - .../mapreduce/MapReduceManagerImpl.java | 818 ----------- .../distexec/mapreduce/MapReduceTask.java | 1229 ----------------- .../infinispan/distexec/mapreduce/Mapper.java | 34 - .../distexec/mapreduce/Reducer.java | 35 - .../distexec/mapreduce/SecurityActions.java | 69 - .../distexec/mapreduce/package-info.java | 6 - .../spi/DefaultMapReduceTaskLifecycle.java | 31 - .../mapreduce/spi/MapReduceTaskLifecycle.java | 16 - .../spi/MapReduceTaskLifecycleService.java | 82 -- .../compat/BaseTypeConverterInterceptor.java | 4 +- .../marshall/core/ExternalizerTable.java | 3 - .../exts/CacheRpcCommandExternalizer.java | 6 +- .../java/org/infinispan/util/logging/Log.java | 3 - .../org/infinispan/api/SimpleCacheTest.java | 6 - .../BaseLargeWordCountMapReduceTest.java | 219 --- .../mapreduce/BaseWordCountMapReduceTest.java | 437 ------ .../infinispan/distexec/mapreduce/Book.java | 27 - .../distexec/mapreduce/BookSearchTest.java | 85 -- .../BookStoreAsBinarySearchTest.java | 26 - .../BookStoreAsBinaryWithStoreSearchTest.java | 34 - ...tModeDistributedTwoNodesMapReduceTest.java | 36 - .../DistributedFourNodesMapReduceTest.java | 30 - ...termediateCacheFourNodesMapReduceTest.java | 68 - ...iateSharedCacheFourNodesMapReduceTest.java | 68 - ...utedSharedCacheFourNodesMapReduceTest.java | 30 - ...butedSharedCacheTwoNodesMapReduceTest.java | 48 - .../DistributedTwoNodesMapReduceTest.java | 29 - ...argeDistributedFourNodesMapReduceTest.java | 42 - ...tedFourNodesMapReduceWithEvictionTest.java | 34 - ...utedSharedCacheFourNodesMapReduceTest.java | 32 - ...cheFourNodesMapReduceWithEvictionTest.java | 35 - ...butedSharedCacheTwoNodesMapReduceTest.java | 31 - ...LargeDistributedTwoNodesMapReduceTest.java | 31 - .../mapreduce/LocalMapReduceTest.java | 165 --- .../mapreduce/LocalSimpleBookSearchTest.java | 31 - .../ReplicatedFourNodesMapReduceTest.java | 22 - .../ReplicatedTwoNodesMapReduceTest.java | 22 - .../mapreduce/SimpleBookSearchTest.java | 20 - .../SimpleFourNodesMapReduceTest.java | 21 - .../SimpleMapReduceTaskTimeoutTest.java | 241 ---- .../SimpleTwoNodesMapReduceTest.java | 367 ----- .../TopologyAwareTwoNodesMapReduceTest.java | 52 - .../TwoNodesWithCacheLoaderMapReduceTest.java | 24 - .../TwoNodesWithStoreMapReduceTest.java | 102 -- .../remoting/AsynchronousInvocationTest.java | 4 +- .../DistributedStreamRehashStressTest.java | 2 - .../util/mocks/ControlledCommandFactory.java | 14 - .../demo/mapreduce/WordCountCollator.java | 52 - .../demo/mapreduce/WordCountDemo.java | 113 -- .../demo/mapreduce/WordCountMapper.java | 35 - .../WordCountMapperEmitPerValue.java | 53 - .../demo/mapreduce/WordCountReducer.java | 23 - .../main/asciidoc/upgrading/upgrading.adoc | 4 + .../chapter-6-Querying_Infinispan.adoc | 7 +- .../user_guide/chapter-63-Streams.adoc | 8 +- .../chapter-66-Map_Reduce_framework.adoc | 240 ---- .../user_guide/chapter-75-Security.adoc | 2 +- .../user_guide/chapter-77-Simple_Cache.adoc | 3 +- .../user_guide/chapter-78-Scripting.adoc | 7 - .../main/asciidoc/user_guide/user_guide.adoc | 1 - .../all/embedded/EmbeddedAllTest.java | 35 - .../mapreduce/BaseWordCountMapReduceTest.java | 100 -- .../DistributedTwoNodesMapReduceTest.java | 46 - .../security/ExecutionAuthorizationTest.java | 117 +- .../scripting/impl/CollatorScript.java | 38 - .../scripting/impl/ExecutionMode.java | 10 +- .../scripting/impl/MapReduceRunner.java | 45 - .../scripting/impl/MapperScript.java | 45 - .../scripting/impl/ReducerScript.java | 49 - .../impl/ScriptingMapReduceTaskLifecycle.java | 49 - .../scripting/ClusteredScriptingTest.java | 23 - .../src/test/resources/wordCountCollator.js | 12 - .../src/test/resources/wordCountMapper.js | 9 - .../src/test/resources/wordCountReducer.js | 6 - .../server/hotrod/AbstractEncoder1x.scala | 3 +- .../infinispan/server/hotrod/Encoder2x.scala | 3 +- .../server/hotrod/util/BulkUtil.java | 98 +- .../src/test/resources/stream_serverTask.js | 2 +- 96 files changed, 64 insertions(+), 6966 deletions(-) delete mode 100644 cdi/embedded/src/main/java/org/infinispan/cdi/DelegatingMapReduceTaskLifecycle.java delete mode 100644 cdi/embedded/src/main/resources/META-INF/services/org.infinispan.distexec.mapreduce.spi.MapReduceTaskLifecycle delete mode 100644 cdi/embedded/src/test/java/org/infinispan/cdi/test/distexec/WordCountMapReduceCDITest.java delete mode 100644 core/src/main/java/org/infinispan/commands/read/MapCombineCommand.java delete mode 100644 core/src/main/java/org/infinispan/commands/read/ReduceCommand.java delete mode 100644 core/src/main/java/org/infinispan/distexec/mapreduce/Collator.java delete mode 100644 core/src/main/java/org/infinispan/distexec/mapreduce/Collector.java delete mode 100644 core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceCacheLoaderTask.java delete mode 100644 core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceException.java delete mode 100644 core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceManager.java delete mode 100644 core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceManagerFactory.java delete mode 100644 core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceManagerImpl.java delete mode 100644 core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceTask.java delete mode 100644 core/src/main/java/org/infinispan/distexec/mapreduce/Mapper.java delete mode 100644 core/src/main/java/org/infinispan/distexec/mapreduce/Reducer.java delete mode 100644 core/src/main/java/org/infinispan/distexec/mapreduce/SecurityActions.java delete mode 100644 core/src/main/java/org/infinispan/distexec/mapreduce/package-info.java delete mode 100644 core/src/main/java/org/infinispan/distexec/mapreduce/spi/DefaultMapReduceTaskLifecycle.java delete mode 100644 core/src/main/java/org/infinispan/distexec/mapreduce/spi/MapReduceTaskLifecycle.java delete mode 100644 core/src/main/java/org/infinispan/distexec/mapreduce/spi/MapReduceTaskLifecycleService.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/BaseLargeWordCountMapReduceTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/BaseWordCountMapReduceTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/Book.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/BookSearchTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/BookStoreAsBinarySearchTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/BookStoreAsBinaryWithStoreSearchTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/CompatModeDistributedTwoNodesMapReduceTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/DistributedFourNodesMapReduceTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/DistributedIntermediateCacheFourNodesMapReduceTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/DistributedIntermediateSharedCacheFourNodesMapReduceTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/DistributedSharedCacheFourNodesMapReduceTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/DistributedSharedCacheTwoNodesMapReduceTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/DistributedTwoNodesMapReduceTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedFourNodesMapReduceTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedFourNodesMapReduceWithEvictionTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedSharedCacheFourNodesMapReduceTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedSharedCacheFourNodesMapReduceWithEvictionTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedSharedCacheTwoNodesMapReduceTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedTwoNodesMapReduceTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/LocalMapReduceTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/LocalSimpleBookSearchTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/ReplicatedFourNodesMapReduceTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/ReplicatedTwoNodesMapReduceTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/SimpleBookSearchTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/SimpleFourNodesMapReduceTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/SimpleMapReduceTaskTimeoutTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/SimpleTwoNodesMapReduceTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/TopologyAwareTwoNodesMapReduceTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/TwoNodesWithCacheLoaderMapReduceTest.java delete mode 100644 core/src/test/java/org/infinispan/distexec/mapreduce/TwoNodesWithStoreMapReduceTest.java delete mode 100644 demos/distexec/src/main/java/org/infinispan/demo/mapreduce/WordCountCollator.java delete mode 100644 demos/distexec/src/main/java/org/infinispan/demo/mapreduce/WordCountDemo.java delete mode 100644 demos/distexec/src/main/java/org/infinispan/demo/mapreduce/WordCountMapper.java delete mode 100644 demos/distexec/src/main/java/org/infinispan/demo/mapreduce/WordCountMapperEmitPerValue.java delete mode 100644 demos/distexec/src/main/java/org/infinispan/demo/mapreduce/WordCountReducer.java delete mode 100644 documentation/src/main/asciidoc/user_guide/chapter-66-Map_Reduce_framework.adoc delete mode 100644 integrationtests/osgi/src/test/java/org/infinispan/it/osgi/distexec/mapreduce/BaseWordCountMapReduceTest.java delete mode 100644 integrationtests/osgi/src/test/java/org/infinispan/it/osgi/distexec/mapreduce/DistributedTwoNodesMapReduceTest.java delete mode 100644 scripting/src/main/java/org/infinispan/scripting/impl/CollatorScript.java delete mode 100644 scripting/src/main/java/org/infinispan/scripting/impl/MapReduceRunner.java delete mode 100644 scripting/src/main/java/org/infinispan/scripting/impl/MapperScript.java delete mode 100644 scripting/src/main/java/org/infinispan/scripting/impl/ReducerScript.java delete mode 100644 scripting/src/main/java/org/infinispan/scripting/impl/ScriptingMapReduceTaskLifecycle.java delete mode 100644 scripting/src/test/resources/wordCountCollator.js delete mode 100644 scripting/src/test/resources/wordCountMapper.js delete mode 100644 scripting/src/test/resources/wordCountReducer.js diff --git a/cdi/embedded/src/main/java/org/infinispan/cdi/ContextInputCache.java b/cdi/embedded/src/main/java/org/infinispan/cdi/ContextInputCache.java index 853321ddf81d..e939bb935af4 100644 --- a/cdi/embedded/src/main/java/org/infinispan/cdi/ContextInputCache.java +++ b/cdi/embedded/src/main/java/org/infinispan/cdi/ContextInputCache.java @@ -2,17 +2,16 @@ import org.infinispan.Cache; import org.infinispan.distexec.DistributedExecutorService; -import org.infinispan.distexec.mapreduce.Mapper; /** * ContextInputCache keeps track of {@link Input} cache to be injected into Callables from - * {@link DistributedExecutorService} and {@link Mapper} from {@link MapReduceTask} using CDI + * {@link DistributedExecutorService} using CDI * mechanism. The cache injected will be the cache used to construct - * {@link DistributedExecutorService} and/or {@link MapReduceTask} - * + * {@link DistributedExecutorService} + * * @author Vladimir Blagoejvic * @since 5.2 - * @see InfinispanExtension#registerInputCacheCustomBean(javax.enterprise.inject.spi.AfterBeanDiscovery, + * @see InfinispanExtensionEmbedded#registerInputCacheCustomBean(javax.enterprise.inject.spi.AfterBeanDiscovery, * javax.enterprise.inject.spi.BeanManager) * */ diff --git a/cdi/embedded/src/main/java/org/infinispan/cdi/DelegatingMapReduceTaskLifecycle.java b/cdi/embedded/src/main/java/org/infinispan/cdi/DelegatingMapReduceTaskLifecycle.java deleted file mode 100644 index 10d3d9d38838..000000000000 --- a/cdi/embedded/src/main/java/org/infinispan/cdi/DelegatingMapReduceTaskLifecycle.java +++ /dev/null @@ -1,125 +0,0 @@ -package org.infinispan.cdi; - -import org.infinispan.Cache; -import org.infinispan.cdi.util.CDIHelper; -import org.infinispan.distexec.mapreduce.Mapper; -import org.infinispan.distexec.mapreduce.Reducer; -import org.infinispan.distexec.mapreduce.spi.MapReduceTaskLifecycle; - -import javax.enterprise.context.spi.CreationalContext; -import javax.enterprise.inject.spi.AnnotatedType; -import javax.enterprise.inject.spi.BeanManager; -import javax.enterprise.inject.spi.InjectionTarget; - -public class DelegatingMapReduceTaskLifecycle implements MapReduceTaskLifecycle { - - private final MapReduceTaskLifecycle delegate; - - public DelegatingMapReduceTaskLifecycle() { - delegate = CDIHelper.isCDIAvailable() ? new CDIMapReduceTaskLifecycle() : new NoCDIMapReduceTaskLifecycle(); - } - - @Override - public void onPreExecute(Mapper mapper, Cache inputCache) { - delegate.onPreExecute(mapper, inputCache); - } - - @Override - public void onPostExecute(Mapper mapper) { - delegate.onPostExecute(mapper); - } - - @Override - public void onPreExecute(Reducer reducer, Cache cache) { - delegate.onPreExecute(reducer, cache); - } - - @Override - public void onPostExecute(Reducer reducer) { - delegate.onPostExecute(reducer); - } - - static class NoCDIMapReduceTaskLifecycle implements MapReduceTaskLifecycle { - - @Override - public void onPreExecute(Mapper mapper, Cache inputCache) { - } - - @Override - public void onPostExecute(Mapper mapper) { - } - - @Override - public void onPreExecute(Reducer reducer, Cache inputCache) { - } - - @Override - public void onPostExecute(Reducer reducer) { - } - - } - - static class CDIMapReduceTaskLifecycle implements MapReduceTaskLifecycle { - - @Override - @SuppressWarnings("unchecked") - public void onPreExecute(Mapper mapper, Cache inputCache) { - BeanManager bm = CDIHelper.getBeanManager(); - if (bm == null) - return; - ContextInputCache.set(inputCache); - Class> clazz = (Class>) mapper.getClass(); - AnnotatedType> type = bm.createAnnotatedType(clazz); - InjectionTarget> it = bm.createInjectionTarget(type); - CreationalContext> ctx = bm.createCreationalContext(null); - it.inject(mapper, ctx); - it.postConstruct(mapper); - } - - @Override - @SuppressWarnings("unchecked") - public void onPostExecute(Mapper mapper) { - try { - BeanManager bm = CDIHelper.getBeanManager(); - if (bm == null) - return; - Class> clazz = (Class>) mapper.getClass(); - AnnotatedType> type = bm.createAnnotatedType(clazz); - InjectionTarget> it = bm.createInjectionTarget(type); - it.preDestroy(mapper); - it.dispose(mapper); - } finally { - ContextInputCache.clean(); - } - } - - @Override - @SuppressWarnings("unchecked") - public void onPreExecute(Reducer reducer, Cache inputCache) { - BeanManager bm = CDIHelper.getBeanManager(); - if (bm == null) - return; - Class> clazz = (Class>) reducer.getClass(); - AnnotatedType> type = bm.createAnnotatedType(clazz); - InjectionTarget> it = bm.createInjectionTarget(type); - CreationalContext> ctx = bm.createCreationalContext(null); - it.inject(reducer, ctx); - it.postConstruct(reducer); - } - - @Override - @SuppressWarnings("unchecked") - public void onPostExecute(Reducer reducer) { - BeanManager bm = CDIHelper.getBeanManager(); - if (bm == null) - return; - Class> clazz = (Class>) reducer.getClass(); - AnnotatedType> type = bm.createAnnotatedType(clazz); - InjectionTarget> it = bm.createInjectionTarget(type); - it.preDestroy(reducer); - it.dispose(reducer); - } - - } - -} diff --git a/cdi/embedded/src/main/java/org/infinispan/cdi/Input.java b/cdi/embedded/src/main/java/org/infinispan/cdi/Input.java index 081825cf382b..9f38e34442f7 100644 --- a/cdi/embedded/src/main/java/org/infinispan/cdi/Input.java +++ b/cdi/embedded/src/main/java/org/infinispan/cdi/Input.java @@ -1,7 +1,6 @@ package org.infinispan.cdi; import org.infinispan.distexec.DefaultExecutorService; -import org.infinispan.distexec.mapreduce.MapReduceTask; import javax.inject.Qualifier; import java.lang.annotation.Documented; @@ -13,7 +12,7 @@ /** * Qualifier indicating the injected Cache should be the input Cache used to create - * {@link DefaultExecutorService} or {@link MapReduceTask} + * {@link DefaultExecutorService} * * @author Vladimir Blagojevic * @since 5.2 diff --git a/cdi/embedded/src/main/resources/META-INF/services/org.infinispan.distexec.mapreduce.spi.MapReduceTaskLifecycle b/cdi/embedded/src/main/resources/META-INF/services/org.infinispan.distexec.mapreduce.spi.MapReduceTaskLifecycle deleted file mode 100644 index a3bd5cc781dc..000000000000 --- a/cdi/embedded/src/main/resources/META-INF/services/org.infinispan.distexec.mapreduce.spi.MapReduceTaskLifecycle +++ /dev/null @@ -1 +0,0 @@ -org.infinispan.cdi.DelegatingMapReduceTaskLifecycle diff --git a/cdi/embedded/src/test/java/org/infinispan/cdi/test/distexec/WordCountMapReduceCDITest.java b/cdi/embedded/src/test/java/org/infinispan/cdi/test/distexec/WordCountMapReduceCDITest.java deleted file mode 100644 index 9eb3466141cf..000000000000 --- a/cdi/embedded/src/test/java/org/infinispan/cdi/test/distexec/WordCountMapReduceCDITest.java +++ /dev/null @@ -1,128 +0,0 @@ -package org.infinispan.cdi.test.distexec; - -import org.infinispan.Cache; -import org.infinispan.cdi.Input; -import org.infinispan.cdi.test.DefaultTestEmbeddedCacheManagerProducer; -import org.infinispan.distexec.mapreduce.BaseWordCountMapReduceTest; -import org.infinispan.distexec.mapreduce.Collector; -import org.infinispan.distexec.mapreduce.MapReduceTask; -import org.infinispan.distexec.mapreduce.Mapper; -import org.infinispan.distexec.mapreduce.Reducer; -import org.infinispan.distexec.mapreduce.SimpleTwoNodesMapReduceTest; -import org.infinispan.test.MultipleCacheManagersTest; -import org.jboss.arquillian.container.test.api.Deployment; -import org.jboss.shrinkwrap.api.Archive; -import org.testng.Assert; -import org.testng.annotations.Test; - -import javax.inject.Inject; -import java.util.Iterator; -import java.util.Map; -import java.util.StringTokenizer; - -import static org.infinispan.cdi.test.testutil.Deployments.baseDeployment; - -/** - * BaseTest for MapReduceTask - * - * @author Vladimir Blagojevic - */ -@Test(groups = "functional", testName = "cdi.test.distexec.WordCountMapReduceCDITest") -public class WordCountMapReduceCDITest extends MultipleCacheManagersArquillianTest { - - BaseWordCountMapReduceTest delegate; - - public WordCountMapReduceCDITest() { - delegate = new SimpleTwoNodesMapReduceTest(); - } - - @Override - MultipleCacheManagersTest getDelegate() { - return delegate; - } - - @Deployment - public static Archive deployment() { - return baseDeployment().addClass(WordCountMapReduceCDITest.class) - .addClass(DefaultTestEmbeddedCacheManagerProducer.class); - } - - public void testinvokeMapReduceOnSubsetOfKeys() throws Exception { - MapReduceTask task = delegate.invokeMapReduce(new String[] { - "1", "2", "3" }, new WordCountMapper(), new WordCountReducer()); - Map mapReduce = task.execute(); - Integer count = mapReduce.get("Infinispan"); - assert count == 1; - count = mapReduce.get("Boston"); - assert count == 1; - } - - public void testinvokeMapReduceWithInputCacheOnSubsetOfKeys() throws Exception { - MapReduceTask task = delegate.invokeMapReduce(new String[] { - "1", "2", "3" }, new WordCountImpliedInputCacheMapper(), new WordCountReducer()); - Map mapReduce = task.execute(); - Integer count = mapReduce.get("Infinispan"); - assert count == 1; - count = mapReduce.get("Boston"); - assert count == 1; - } - - private static class WordCountMapper implements Mapper { - /** The serialVersionUID */ - private static final long serialVersionUID = -5943370243108735560L; - - @Inject - private Cache cache; - - @Override - public void map(String key, String value, Collector collector) { - Assert.assertNotNull(cache, "Cache not injected into " + this); - StringTokenizer tokens = new StringTokenizer(value); - while (tokens.hasMoreElements()) { - String s = (String) tokens.nextElement(); - collector.emit(s, 1); - } - } - } - - private static class WordCountImpliedInputCacheMapper implements Mapper { - - /** The serialVersionUID */ - private static final long serialVersionUID = 7525403183805551028L; - - @Inject - @Input - private Cache cache; - - @Override - public void map(String key, String value, Collector collector) { - Assert.assertNotNull(cache, "Cache not injected into " + this); - //the right cache injected - Assert.assertTrue(cache.getName().equals("mapreducecache")); - StringTokenizer tokens = new StringTokenizer(value); - while (tokens.hasMoreElements()) { - String s = (String) tokens.nextElement(); - collector.emit(s, 1); - } - } - } - - private static class WordCountReducer implements Reducer { - /** The serialVersionUID */ - private static final long serialVersionUID = 1901016598354633256L; - - @Inject - private Cache cache; - - @Override - public Integer reduce(String key, Iterator iter) { - Assert.assertNotNull(cache, "Cache not injected into " + this); - int sum = 0; - while (iter.hasNext()) { - Integer i = iter.next(); - sum += i; - } - return sum; - } - } -} diff --git a/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/ExecTest.java b/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/ExecTest.java index ef5fbfef99da..c9c266eead6a 100644 --- a/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/ExecTest.java +++ b/client/hotrod-client/src/test/java/org/infinispan/client/hotrod/ExecTest.java @@ -101,22 +101,6 @@ private void populateCache(String cacheName) { clients.get(i % NUM_SERVERS).getCache(cacheName).put(String.format("Key %d", i), String.format("Value %d", i)); } - public void testRemoteMapReduce() throws Exception { - String cacheName = "testRemoteMapReduce"; - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(CacheMode.DIST_SYNC, true); - builder.dataContainer().keyEquivalence(new AnyServerEquivalence()).valueEquivalence(new AnyServerEquivalence()).compatibility().enable().marshaller(new GenericJBossMarshaller()); - defineInAll(cacheName, builder); - RemoteCache cache = clients.get(0).getCache(cacheName); - RemoteCache scriptCache = clients.get(0).getCache(SCRIPT_CACHE); - loadData(cache, "/macbeth.txt"); - loadScript(scriptCache, "/wordCountMapper.js"); - loadScript(scriptCache, "/wordCountReducer.js"); - loadScript(scriptCache, "/wordCountCollator.js"); - Map results = cache.execute("wordCountMapper.js", new HashMap()); - assertEquals(20, results.size()); - assertTrue(results.get("macbeth").equals(Double.valueOf(287))); - } - @Test(enabled = false, description = "Disabling this test until the distributed scripts in DIST mode are fixed - ISPN-6173") public void testRemoteMapReduceWithStreams() throws Exception { String cacheName = "testRemoteMapReduce_Streams"; diff --git a/core/src/main/java/org/infinispan/commands/CommandsFactory.java b/core/src/main/java/org/infinispan/commands/CommandsFactory.java index b78f39190d1e..e8eb7eb7c003 100644 --- a/core/src/main/java/org/infinispan/commands/CommandsFactory.java +++ b/core/src/main/java/org/infinispan/commands/CommandsFactory.java @@ -18,8 +18,6 @@ import org.infinispan.commands.read.GetCacheEntryCommand; import org.infinispan.commands.read.GetKeyValueCommand; import org.infinispan.commands.read.KeySetCommand; -import org.infinispan.commands.read.MapCombineCommand; -import org.infinispan.commands.read.ReduceCommand; import org.infinispan.commands.read.SizeCommand; import org.infinispan.commands.remote.ClusteredGetAllCommand; import org.infinispan.commands.remote.ClusteredGetCommand; @@ -49,8 +47,6 @@ import org.infinispan.commons.api.functional.EntryView.ReadWriteEntryView; import org.infinispan.commons.api.functional.EntryView.WriteEntryView; import org.infinispan.context.Flag; -import org.infinispan.distexec.mapreduce.Mapper; -import org.infinispan.distexec.mapreduce.Reducer; import org.infinispan.factories.scopes.Scope; import org.infinispan.factories.scopes.Scopes; import org.infinispan.functional.impl.Params; @@ -362,28 +358,6 @@ public interface CommandsFactory { */ DistributedExecuteCommand buildDistributedExecuteCommand(Callable callable, Address sender, Collection keys); - /** - * Builds a MapCombineCommand used for migration and map phase execution of MapReduce tasks. - * - * @param m Mapper for MapReduceTask - * @param r Combiner for MapReduceTask - * @param keys keys used in MapReduceTask - * @return created MapCombineCommand - */ - MapCombineCommand buildMapCombineCommand( - String taskId, Mapper m, Reducer r, - Collection keys); - - /** - * Builds a ReduceCommand used for migration and reduce phase execution of MapReduce tasks. - * - * @param r Reducer for MapReduceTask - * @param keys keys used in MapReduceTask - * @return created ReduceCommand - */ - ReduceCommand buildReduceCommand(String taskId, - String destinationCache, Reducer r, Collection keys); - /** * @see GetInDoubtTxInfoCommand */ diff --git a/core/src/main/java/org/infinispan/commands/CommandsFactoryImpl.java b/core/src/main/java/org/infinispan/commands/CommandsFactoryImpl.java index 841436349ce8..0d0ab1231a4e 100644 --- a/core/src/main/java/org/infinispan/commands/CommandsFactoryImpl.java +++ b/core/src/main/java/org/infinispan/commands/CommandsFactoryImpl.java @@ -20,8 +20,6 @@ import org.infinispan.commands.read.GetCacheEntryCommand; import org.infinispan.commands.read.GetKeyValueCommand; import org.infinispan.commands.read.KeySetCommand; -import org.infinispan.commands.read.MapCombineCommand; -import org.infinispan.commands.read.ReduceCommand; import org.infinispan.commands.read.SizeCommand; import org.infinispan.commands.remote.ClusteredGetAllCommand; import org.infinispan.commands.remote.ClusteredGetCommand; @@ -65,9 +63,6 @@ import org.infinispan.container.InternalEntryFactory; import org.infinispan.context.Flag; import org.infinispan.context.InvocationContextFactory; -import org.infinispan.distexec.mapreduce.MapReduceManager; -import org.infinispan.distexec.mapreduce.Mapper; -import org.infinispan.distexec.mapreduce.Reducer; import org.infinispan.distribution.DistributionManager; import org.infinispan.distribution.group.GroupManager; import org.infinispan.factories.KnownComponentNames; @@ -155,7 +150,6 @@ public class CommandsFactoryImpl implements CommandsFactory { private StateConsumer stateConsumer; private LockManager lockManager; private InternalEntryFactory entryFactory; - private MapReduceManager mapReduceManager; private StateTransferManager stateTransferManager; private BackupSender backupSender; private CancellationService cancellationService; @@ -177,7 +171,7 @@ public void setupDependencies(DataContainer container, CacheNotifier moduleCommandInitializers, RecoveryManager recoveryManager, StateProvider stateProvider, StateConsumer stateConsumer, - LockManager lockManager, InternalEntryFactory entryFactory, MapReduceManager mapReduceManager, + LockManager lockManager, InternalEntryFactory entryFactory, StateTransferManager stm, BackupSender backupSender, CancellationService cancellationService, TimeService timeService, XSiteStateProvider xSiteStateProvider, XSiteStateConsumer xSiteStateConsumer, XSiteStateTransferManager xSiteStateTransferManager, @@ -198,7 +192,6 @@ public void setupDependencies(DataContainer container, CacheNotifier DistributedExecuteCommand buildDistributedExecuteCommand(Callable< return new DistributedExecuteCommand(cacheName, keys, callable); } - @Override - public MapCombineCommand buildMapCombineCommand( - String taskId, Mapper m, Reducer r, - Collection keys) { - return new MapCombineCommand(taskId, m, r, cacheName, keys); - } - @Override public GetInDoubtTxInfoCommand buildGetInDoubtTxInfoCommand() { return new GetInDoubtTxInfoCommand(cacheName); @@ -619,12 +597,6 @@ public CreateCacheCommand buildCreateCacheCommand(String cacheNameToCreate, Stri return new CreateCacheCommand(cacheName, cacheNameToCreate, cacheConfigurationName, size); } - @Override - public ReduceCommand buildReduceCommand(String taskId, - String destintationCache, Reducer r, Collection keys) { - return new ReduceCommand(taskId, r, destintationCache, keys); - } - @Override public CancelCommand buildCancelCommandCommand(UUID commandUUID) { return new CancelCommand(cacheName, commandUUID); diff --git a/core/src/main/java/org/infinispan/commands/CreateCacheCommand.java b/core/src/main/java/org/infinispan/commands/CreateCacheCommand.java index 93f10ed2bbbf..ed6f793dfd4a 100644 --- a/core/src/main/java/org/infinispan/commands/CreateCacheCommand.java +++ b/core/src/main/java/org/infinispan/commands/CreateCacheCommand.java @@ -1,18 +1,9 @@ package org.infinispan.commands; -import java.io.IOException; -import java.io.ObjectInput; -import java.io.ObjectOutput; -import java.util.concurrent.TimeUnit; - import org.infinispan.Cache; import org.infinispan.commands.remote.BaseRpcCommand; -import org.infinispan.commons.marshall.MarshallUtil; -import org.infinispan.configuration.cache.CacheMode; import org.infinispan.configuration.cache.Configuration; -import org.infinispan.configuration.cache.ConfigurationBuilder; import org.infinispan.context.InvocationContext; -import org.infinispan.distexec.mapreduce.MapReduceTask; import org.infinispan.factories.ComponentRegistry; import org.infinispan.manager.EmbeddedCacheManager; import org.infinispan.statetransfer.StateTransferLock; @@ -23,6 +14,11 @@ import org.infinispan.util.logging.Log; import org.infinispan.util.logging.LogFactory; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.concurrent.TimeUnit; + /** * Command to create/start a cache on a subset of Infinispan cluster nodes * @author Vladimir Blagojevic @@ -70,16 +66,9 @@ public Object perform(InvocationContext ctx) throws Throwable { Configuration cacheConfig = cacheManager.getCacheConfiguration(cacheConfigurationName); if (cacheConfig == null) { - // Special case for the default temporary cache, which may or may not have been defined by the user - if (MapReduceTask.DEFAULT_TMP_CACHE_CONFIGURATION_NAME.equals(cacheConfigurationName)) { - cacheConfig = new ConfigurationBuilder().unsafe().unreliableReturnValues(true).clustering() - .cacheMode(CacheMode.DIST_SYNC).hash().numOwners(2).sync().build(); log.debugf( - "Using default tmp cache configuration, defined as ", cacheNameToCreate); - } else { - throw new IllegalStateException( - "Cache configuration " + cacheConfigurationName + " is not defined on node " + - this.cacheManager.getAddress()); - } + throw new IllegalStateException( + "Cache configuration " + cacheConfigurationName + " is not defined on node " + + this.cacheManager.getAddress()); } cacheManager.defineConfiguration(cacheNameToCreate, cacheConfig); diff --git a/core/src/main/java/org/infinispan/commands/RemoteCommandsFactory.java b/core/src/main/java/org/infinispan/commands/RemoteCommandsFactory.java index 2bb6e4916b79..e9e7d4497d2c 100644 --- a/core/src/main/java/org/infinispan/commands/RemoteCommandsFactory.java +++ b/core/src/main/java/org/infinispan/commands/RemoteCommandsFactory.java @@ -13,8 +13,6 @@ import org.infinispan.commands.read.DistributedExecuteCommand; import org.infinispan.commands.read.GetCacheEntryCommand; import org.infinispan.commands.read.GetKeyValueCommand; -import org.infinispan.commands.read.MapCombineCommand; -import org.infinispan.commands.read.ReduceCommand; import org.infinispan.commands.remote.CacheRpcCommand; import org.infinispan.commands.remote.ClusteredGetAllCommand; import org.infinispan.commands.remote.ClusteredGetCommand; @@ -261,12 +259,6 @@ public CacheRpcCommand fromStream(byte id, byte type, String cacheName) { case GetInDoubtTransactionsCommand.COMMAND_ID: command = new GetInDoubtTransactionsCommand(cacheName); break; - case MapCombineCommand.COMMAND_ID: - command = new MapCombineCommand(cacheName); - break; - case ReduceCommand.COMMAND_ID: - command = new ReduceCommand(cacheName); - break; case DistributedExecuteCommand.COMMAND_ID: command = new DistributedExecuteCommand(cacheName); break; diff --git a/core/src/main/java/org/infinispan/commands/read/MapCombineCommand.java b/core/src/main/java/org/infinispan/commands/read/MapCombineCommand.java deleted file mode 100644 index c10a21f14614..000000000000 --- a/core/src/main/java/org/infinispan/commands/read/MapCombineCommand.java +++ /dev/null @@ -1,230 +0,0 @@ -package org.infinispan.commands.read; - -import java.io.IOException; -import java.io.ObjectInput; -import java.io.ObjectOutput; -import java.util.Collection; -import java.util.HashSet; -import java.util.Set; -import java.util.UUID; - -import org.infinispan.commands.CancellableCommand; -import org.infinispan.commands.remote.BaseRpcCommand; -import org.infinispan.commons.marshall.MarshallUtil; -import org.infinispan.context.InvocationContext; -import org.infinispan.distexec.mapreduce.Collector; -import org.infinispan.distexec.mapreduce.MapReduceManager; -import org.infinispan.distexec.mapreduce.Mapper; -import org.infinispan.distexec.mapreduce.Reducer; - - -/** - * MapCombineCommand is a container to migrate {@link Mapper} and {@link Reducer} which is a - * combiner to a remote Infinispan node where it will get executed and return the result to an - * invoking/master node. - * - * @author Vladimir Blagojevic - * @since 5.0 - */ -public class MapCombineCommand extends BaseRpcCommand implements CancellableCommand { - public static final int COMMAND_ID = 30; - private Set keys = new HashSet(); - private Mapper mapper; - private Reducer combiner; - private String taskId; - private boolean reducePhaseDistributed; - private boolean useIntermediateSharedCache; - private MapReduceManager mrManager; - private UUID uuid; - private String intermediateCacheName; - private int maxCollectorSize; - - public MapCombineCommand() { - super(null); // For command id uniqueness test - } - - public MapCombineCommand(String cacheName) { - super(cacheName); - } - - public MapCombineCommand(String taskId, Mapper mapper, - Reducer combiner, String cacheName, Collection inputKeys) { - super(cacheName); - this.taskId = taskId; - if (inputKeys != null && !inputKeys.isEmpty()) { - keys.addAll(inputKeys); - } - this.mapper = mapper; - this.combiner = combiner; - this.uuid = UUID.randomUUID(); - } - - public void init(MapReduceManager mrManager) { - this.mrManager = mrManager; - } - - /** - * Performs invocation of mapping phase and local combine phase on assigned Infinispan node - * - * @param context - * invocation context - * @return Map of intermediate key value pairs - */ - @Override - public Object perform(InvocationContext context) throws Throwable { - if (isReducePhaseDistributed()) - return mrManager.mapAndCombineForDistributedReduction(this); - else - return mrManager.mapAndCombineForLocalReduction(this); - } - - public boolean isUseIntermediateSharedCache() { - return useIntermediateSharedCache; - } - - public void setUseIntermediateSharedCache(boolean useSharedTmpCache) { - this.useIntermediateSharedCache = useSharedTmpCache; - } - - public boolean isReducePhaseDistributed() { - return reducePhaseDistributed; - } - - public void setReducePhaseDistributed(boolean reducePhaseDistributed) { - this.reducePhaseDistributed = reducePhaseDistributed; - } - - public void setIntermediateCacheName(String intermediateCacheName) { - this.intermediateCacheName = intermediateCacheName; - } - - /** - * Limits Mapper's Collector size to a given value. - *

- * During execution of map/combine phase, number of intermediate keys/values collected in - * Collector could potentially become very large. By limiting size of collector intermediate - * key/values are transferred to intermediate cache in batches before reduce phase is executed. - *

- * The default value for max collector size is 1024. - * - * @param size - * the number of key/value pairs for one batch transfer - * - * @see Mapper#map(Object, Object, Collector) - */ - public void setMaxCollectorSize(int size) { - if (size <= 0) - throw new IllegalArgumentException("Invalid size " + size); - maxCollectorSize = size; - } - - public int getMaxCollectorSize() { - return maxCollectorSize; - } - - public Set getKeys() { - return keys; - } - - public Mapper getMapper() { - return mapper; - } - - public Reducer getCombiner() { - return combiner; - } - - public boolean hasCombiner(){ - return getCombiner() != null; - } - - public String getTaskId() { - return taskId; - } - - public String getIntermediateCacheName() { - return intermediateCacheName; - } - - @Override - public byte getCommandId() { - return COMMAND_ID; - } - - @Override - public UUID getUUID() { - return uuid; - } - - @Override - public void writeTo(ObjectOutput output) throws IOException { - output.writeUTF(taskId); - MarshallUtil.marshallCollection(keys, output); - output.writeObject(mapper); - output.writeObject(combiner); - output.writeBoolean(reducePhaseDistributed); - output.writeBoolean(useIntermediateSharedCache); - MarshallUtil.marshallUUID(uuid, output, false); - output.writeUTF(intermediateCacheName); - output.writeInt(maxCollectorSize); - } - - @Override - public void readFrom(ObjectInput input) throws IOException, ClassNotFoundException { - taskId = input.readUTF(); - keys = MarshallUtil.unmarshallCollectionUnbounded(input, HashSet::new); - mapper = (Mapper) input.readObject(); - combiner = (Reducer) input.readObject(); - reducePhaseDistributed = input.readBoolean(); - useIntermediateSharedCache = input.readBoolean(); - uuid = MarshallUtil.unmarshallUUID(input, false); - intermediateCacheName = input.readUTF(); - maxCollectorSize = input.readInt(); - } - - @Override - public boolean isReturnValueExpected() { - return true; - } - - @Override - public boolean canBlock() { - return true; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((taskId == null) ? 0 : taskId.hashCode()); - return result; - } - - @SuppressWarnings("rawtypes") - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (!(obj instanceof MapCombineCommand)) { - return false; - } - MapCombineCommand other = (MapCombineCommand) obj; - if (taskId == null) { - if (other.taskId != null) { - return false; - } - } else if (!taskId.equals(other.taskId)) { - return false; - } - return true; - } - - @Override - public String toString() { - return "MapCombineCommand [keys=" + keys + ", taskId=" + taskId + "]"; - } -} diff --git a/core/src/main/java/org/infinispan/commands/read/ReduceCommand.java b/core/src/main/java/org/infinispan/commands/read/ReduceCommand.java deleted file mode 100644 index 97481388ae91..000000000000 --- a/core/src/main/java/org/infinispan/commands/read/ReduceCommand.java +++ /dev/null @@ -1,184 +0,0 @@ -package org.infinispan.commands.read; - -import java.io.IOException; -import java.io.ObjectInput; -import java.io.ObjectOutput; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; -import java.util.UUID; - -import org.infinispan.commands.CancellableCommand; -import org.infinispan.commands.remote.BaseRpcCommand; -import org.infinispan.commons.marshall.MarshallUtil; -import org.infinispan.context.InvocationContext; -import org.infinispan.distexec.mapreduce.MapReduceManager; -import org.infinispan.distexec.mapreduce.Reducer; - -/** - * ReduceCommand is a container to migrate {@link Reducer} to a remote Infinispan node where it will - * get executed and return the result to an invoking/master node. - * - * @author Vladimir Blagojevic - * @since 5.2 - */ -public class ReduceCommand extends BaseRpcCommand implements CancellableCommand { - - public static final int COMMAND_ID = 31; - private Set keys = new HashSet(); - private Reducer reducer; - private String taskId; - private boolean useIntermediateSharedCache; - private MapReduceManager mrManager; - private UUID uuid; - private String resultCacheName; - - private ReduceCommand() { - super(null); // For command id uniqueness test - } - - public ReduceCommand(String cacheName) { - super(cacheName); - } - - public ReduceCommand(String taskId, Reducer reducer, String cacheName, - Collection inputKeys) { - super(cacheName); - this.taskId = taskId; - if (inputKeys != null && !inputKeys.isEmpty()) { - keys.addAll(inputKeys); - } - this.reducer = reducer; - this.uuid = UUID.randomUUID(); - } - - public void init(MapReduceManager mrManager) { - this.mrManager = mrManager; - } - - /** - * Performs invocation of reduce phase on assigned Infinispan node - * - * @param context - * invocation context - * @return Map of intermediate key value pairs - */ - @Override - public Object perform(InvocationContext context) throws Throwable { - if (emitsIntoResultingCache()){ - mrManager.reduce(this, getResultCacheName()); - return Collections.emptyMap(); - } else { - return mrManager.reduce(this); - } - } - - public boolean isUseIntermediateSharedCache() { - return useIntermediateSharedCache; - } - - public void setUseIntermediateSharedCache(boolean useIntermediateSharedCache) { - this.useIntermediateSharedCache = useIntermediateSharedCache; - } - - public boolean emitsIntoResultingCache(){ - return resultCacheName != null && !resultCacheName.isEmpty(); - } - - public String getResultCacheName() { - return resultCacheName; - } - - public void setResultCacheName(String resultCacheName) { - this.resultCacheName = resultCacheName; - } - - public Set getKeys() { - return keys; - } - - public Reducer getReducer() { - return reducer; - } - - public String getTaskId() { - return taskId; - } - - @Override - public byte getCommandId() { - return COMMAND_ID; - } - - @Override - public UUID getUUID() { - return uuid; - } - - @Override - public void writeTo(ObjectOutput output) throws IOException { - output.writeUTF(taskId); - MarshallUtil.marshallCollection(keys, output); - output.writeObject(reducer); - output.writeBoolean(useIntermediateSharedCache); - MarshallUtil.marshallUUID(uuid, output, false); - MarshallUtil.marshallString(resultCacheName, output); - } - - @Override - public void readFrom(ObjectInput input) throws IOException, ClassNotFoundException { - taskId = input.readUTF(); - keys = MarshallUtil.unmarshallCollectionUnbounded(input, HashSet::new); - reducer = (Reducer) input.readObject(); - useIntermediateSharedCache = input.readBoolean(); - uuid = MarshallUtil.unmarshallUUID(input, false); - resultCacheName = MarshallUtil.unmarshallString(input); - } - - @Override - public boolean isReturnValueExpected() { - return true; - } - - @Override - public boolean canBlock() { - return true; - } - - @SuppressWarnings("rawtypes") - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (!(obj instanceof ReduceCommand)) { - return false; - } - ReduceCommand other = (ReduceCommand) obj; - if (taskId == null) { - if (other.taskId != null) { - return false; - } - } else if (!taskId.equals(other.taskId)) { - return false; - } - return true; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((taskId == null) ? 0 : taskId.hashCode()); - return result; - } - - @Override - public String toString() { - return "ReduceCommand [keys=" + keys + ", taskId=" + taskId + "]"; - } -} diff --git a/core/src/main/java/org/infinispan/distexec/mapreduce/Collator.java b/core/src/main/java/org/infinispan/distexec/mapreduce/Collator.java deleted file mode 100644 index 71c14da0f403..000000000000 --- a/core/src/main/java/org/infinispan/distexec/mapreduce/Collator.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import java.util.Map; - -/** - * Collator collates results from Reducers executed on Infinispan cluster and assembles a final - * result returned to an invoker of MapReduceTask. - * - * - * @see MapReduceTask#execute(Collator) - * @see MapReduceTask#executeAsynchronously(Collator) - * @see Reducer - * - * @author Manik Surtani - * @author Vladimir Blagojevic - * - * @since 5.0 - */ -public interface Collator { - - /** - * Collates all reduced results and returns R to invoker of distributed task. - * - * @return final result of distributed task computation - */ - R collate(Map reducedResults); -} diff --git a/core/src/main/java/org/infinispan/distexec/mapreduce/Collector.java b/core/src/main/java/org/infinispan/distexec/mapreduce/Collector.java deleted file mode 100644 index b177d60f4cd8..000000000000 --- a/core/src/main/java/org/infinispan/distexec/mapreduce/Collector.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -/** - * OutputCollector is the intermediate key/value result data output collector given to each {@link Mapper} - * - * @see Mapper#map(Object, Object, Collector) - * - * @author Mircea Markus - * @author Sanne Grinovero - * @since 5.0 - */ -public interface Collector { - - /** - * Intermediate key/value callback used by {@link Mapper} implementor - * - * @param key - * intermediate key - * @param value - * intermediate value - */ - void emit(K key, V value); - -} diff --git a/core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceCacheLoaderTask.java b/core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceCacheLoaderTask.java deleted file mode 100644 index b3f351b04d22..000000000000 --- a/core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceCacheLoaderTask.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.persistence.spi.AdvancedCacheLoader; -import org.infinispan.marshall.core.MarshalledEntry; -import org.infinispan.marshall.core.MarshalledValue; - -/** - * This is an internal class, not intended to be used by clients. - * - * @author Mircea Markus - * @since 6.0 - */ -public class MapReduceCacheLoaderTask implements AdvancedCacheLoader.CacheLoaderTask { - - final Mapper mapper; - final Collector collector; - - public MapReduceCacheLoaderTask(Mapper mapper, Collector collector) { - this.mapper = mapper; - this.collector = collector; - } - - @Override - public void processEntry(MarshalledEntry marshalledEntry, AdvancedCacheLoader.TaskContext taskContext) { - mapper.map(marshalledEntry.getKey(), getValue(marshalledEntry), collector); - } - - @SuppressWarnings("unchecked") - private VIn getValue(MarshalledEntry marshalledEntry) { - Object loadedValue = marshalledEntry.getValue(); - if (loadedValue instanceof MarshalledValue) { - return (VIn)((MarshalledValue) loadedValue).get(); - } else { - return (VIn) loadedValue; - } - } -} diff --git a/core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceException.java b/core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceException.java deleted file mode 100644 index 0774f5670546..000000000000 --- a/core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceException.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.commons.CacheException; - -/** - * An exception indicating Map/Reduce job failure - * - * @author Vladimir Blagojevic - * @since 7.0 - */ -public class MapReduceException extends CacheException { - - private static final long serialVersionUID = -1699361066674350664L; - - public MapReduceException(Throwable cause) { - super(cause); - } - - public MapReduceException(String message, Throwable cause) { - super(message, cause); - } - - public MapReduceException(String message) { - super(message); - } -} diff --git a/core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceManager.java b/core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceManager.java deleted file mode 100644 index 9489c284fe98..000000000000 --- a/core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceManager.java +++ /dev/null @@ -1,96 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ExecutorService; - -import org.infinispan.commands.read.MapCombineCommand; -import org.infinispan.commands.read.ReduceCommand; -import org.infinispan.distribution.DistributionManager; -import org.infinispan.remoting.transport.Address; - -/** - * MapReduceManager is an internal Infinispan component receiving map/reduce invocations arriving - * from remote Infinispan nodes. - *

- * - * This interface should never be implemented by clients. - * - * @author Vladimir Blagojevic - * @since 5.2 - */ -public interface MapReduceManager { - - /** - * Invoked when MapCombineCommand arrives to a target Infinispan node and returns set of - * intermediate keys that needs to be reduced. - * - * @param mcc MapCombineCommand sent from MapReduceTask - * @return a set of intermediate keys to be reduced distributively - */ - Set mapAndCombineForDistributedReduction( - MapCombineCommand mcc) throws InterruptedException; - - /** - * Invoked when MapCombineCommand arrives to a target Infinispan node and returns map of resulting - * values that need to be reduced. - *

- * Unlike {@link #mapAndCombineForDistributedReduction(MapCombineCommand)} - * implementations should return a map Map> which should be ready for reduction - * at master Infinispan node - *

- * - * @param mcc MapCombineCommand sent from MapReduceTask - * @return a map Map> which should be ready for reduction - * at master Infinispan node - */ - Map> mapAndCombineForLocalReduction( - MapCombineCommand mcc) throws InterruptedException; - - /** - * Invoked when ReduceCommand arrives to a target Infinispan node. Implementations should return - * a map of reduced output keys and values to be returned to invoker of MapReduceTask - * - * @param reducer ReduceCommand sent from MapReduceTask - * @return map of reduced output keys and values returned to MapReduceTask - */ - Map reduce(ReduceCommand reducer) throws InterruptedException; - - /** - * Invoked when ReduceCommand arrives to a target Infinispan node. However, instead of returning - * a resulting Map to master node, reduce command emits results of reduce phase to a - * specified resulting cache. - * - * @param reducer ReduceCommand sent from MapReduceTask - * @param resultCache result cache to store results of reduce phase - * @return map of reduced output keys and values returned to MapReduceTask - */ - void reduce(ReduceCommand reducer, String resultCache) - throws InterruptedException; - - /** - * Maps Map/Reduce task intermediate or input keys to nodes on Infinispan cluster - * - * - * @param dm - * distribution manager to use for locating keys on hash wheel - * @param taskId - * id of the map/reduce task - * @param keysToMap - * list of input keys to locate in the cluster - * @return map where each key is an Address in the cluster and value are the keys mapped to that - * Address - */ - Map> mapKeysToNodes(DistributionManager dm, String taskId, - Collection keysToMap); - - /** - * ExecutorService provided for local task execution - * - * @return {@link ExecutorService} for local tasks - */ - ExecutorService getExecutorService(); - -} diff --git a/core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceManagerFactory.java b/core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceManagerFactory.java deleted file mode 100644 index e039eb9d519c..000000000000 --- a/core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceManagerFactory.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.factories.AbstractNamedCacheComponentFactory; -import org.infinispan.factories.AutoInstantiableFactory; -import org.infinispan.factories.annotations.DefaultFactoryFor; - -/** - * MapReduceManagerFactory is a default factory class for {@link MapReduceManager}. - *

- * This is an internal class, not intended to be used by clients. - * @author Vladimir Blagojevic - * @since 5.2 - */ -@DefaultFactoryFor(classes={MapReduceManager.class}) -public class MapReduceManagerFactory extends AbstractNamedCacheComponentFactory implements - AutoInstantiableFactory { - - @SuppressWarnings("unchecked") - @Override - public T construct(Class componentType) { - T result = (T) new MapReduceManagerImpl(); - return result; - } -} diff --git a/core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceManagerImpl.java b/core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceManagerImpl.java deleted file mode 100644 index a3da5b026f9e..000000000000 --- a/core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceManagerImpl.java +++ /dev/null @@ -1,818 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.Cache; -import org.infinispan.atomic.Delta; -import org.infinispan.atomic.DeltaAware; -import org.infinispan.commands.read.MapCombineCommand; -import org.infinispan.commands.read.ReduceCommand; -import org.infinispan.commons.CacheException; -import org.infinispan.commons.marshall.AbstractExternalizer; -import org.infinispan.commons.util.CollectionFactory; -import org.infinispan.commons.util.Util; -import org.infinispan.configuration.cache.Configuration; -import org.infinispan.container.DataContainer; -import org.infinispan.container.entries.InternalCacheEntry; -import org.infinispan.context.Flag; -import org.infinispan.distexec.mapreduce.spi.MapReduceTaskLifecycleService; -import org.infinispan.distribution.DistributionManager; -import org.infinispan.factories.annotations.ComponentName; -import org.infinispan.factories.annotations.Inject; -import org.infinispan.filter.CollectionKeyFilter; -import org.infinispan.filter.CompositeKeyFilter; -import org.infinispan.filter.KeyFilter; -import org.infinispan.interceptors.locking.ClusteringDependentLogic; -import org.infinispan.persistence.manager.PersistenceManager; -import org.infinispan.persistence.PrimaryOwnerFilter; -import org.infinispan.persistence.spi.AdvancedCacheLoader; -import org.infinispan.persistence.spi.AdvancedCacheLoader.TaskContext; -import org.infinispan.manager.EmbeddedCacheManager; -import org.infinispan.marshall.core.Ids; -import org.infinispan.marshall.core.MarshalledEntry; -import org.infinispan.marshall.core.MarshalledValue; -import org.infinispan.remoting.transport.Address; -import org.infinispan.util.TimeService; -import org.infinispan.util.logging.Log; -import org.infinispan.util.logging.LogFactory; - -import java.io.IOException; -import java.io.ObjectInput; -import java.io.ObjectOutput; -import java.io.Serializable; -import java.util.*; -import java.util.Map.Entry; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiConsumer; - -import static org.infinispan.factories.KnownComponentNames.ASYNC_TRANSPORT_EXECUTOR; - -/** - * Default implementation of {@link MapReduceManager}. - *

- * - * - * This is an internal class, not intended to be used by clients. - * @author Vladimir Blagojevic - * @since 5.2 - */ -public class MapReduceManagerImpl implements MapReduceManager { - - private static final Log log = LogFactory.getLog(MapReduceManagerImpl.class); - private static final boolean trace = log.isTraceEnabled(); - private ClusteringDependentLogic cdl; - private EmbeddedCacheManager cacheManager; - private PersistenceManager persistenceManager; - private ExecutorService executorService; - private TimeService timeService; - private int chunkSize; - - MapReduceManagerImpl() { - } - - @Inject - public void init(EmbeddedCacheManager cacheManager, PersistenceManager persistenceManager, - @ComponentName(ASYNC_TRANSPORT_EXECUTOR) ExecutorService asyncTransportExecutor, - ClusteringDependentLogic cdl, TimeService timeService, Configuration configuration) { - this.cacheManager = cacheManager; - this.persistenceManager = persistenceManager; - this.cdl = cdl; - this.executorService = asyncTransportExecutor; - this.timeService = timeService; - this.chunkSize = configuration.clustering().stateTransfer().chunkSize(); - } - - @Override - public ExecutorService getExecutorService() { - return executorService; - } - - @Override - public Map> mapAndCombineForLocalReduction( - MapCombineCommand mcc) throws InterruptedException { - CollectableCollector collector = map(mcc); - combine(mcc, collector); - return collector.collectedValues(); - } - - @Override - public Set mapAndCombineForDistributedReduction( - MapCombineCommand mcc) throws InterruptedException { - try { - return mapAndCombine(mcc); - } catch (Exception e) { - throw new CacheException(e); - } - } - - @Override - public Map reduce(ReduceCommand reduceCommand) throws InterruptedException { - final Map result = CollectionFactory.makeConcurrentMap(256); - reduce(reduceCommand, result); - return result; - } - - @Override - public void reduce(ReduceCommand reduceCommand, String resultCache) throws InterruptedException{ - Cache cache = cacheManager.getCache(resultCache); - reduce(reduceCommand, cache); - } - - protected void reduce(ReduceCommand reduceCommand, final Map result) - throws InterruptedException { - final Set keys = reduceCommand.getKeys(); - final String taskId = reduceCommand.getTaskId(); - boolean noInputKeys = keys == null || keys.isEmpty(); - - if (noInputKeys) { - //illegal state, raise exception - throw new IllegalStateException("Reduce phase of MapReduceTask " + taskId + " on node " + cdl.getAddress() - + " executed with empty input keys"); - } else { - final Reducer reducer = reduceCommand.getReducer(); - final boolean sharedTmpCacheUsed = reduceCommand.isUseIntermediateSharedCache(); - MapReduceTaskLifecycleService taskLifecycleService = MapReduceTaskLifecycleService.getInstance(); - log.tracef("For m/r task %s invoking %s at %s", taskId, reduceCommand, cdl.getAddress()); - long start = trace ? timeService.time() : 0; - try { - Cache, List> cache = cacheManager.getCache(reduceCommand.getCacheName()); - taskLifecycleService.onPreExecute(reducer, cache); - KeyFilter> filter = new IntermediateKeyFilter(taskId, !sharedTmpCacheUsed); - //iterate all tmp cache entries in memory, do it in parallel - DataContainer, List> dc = cache.getAdvancedCache().getDataContainer(); - dc.executeTask(filter, new DataContainerTask, List>() { - @Override - public void accept(IntermediateKey k, InternalCacheEntry, List> v) { - KOut key = k.getKey(); - //resolve Iterable for iterated key stored in tmp cache - Iterable value = getValue(v); - if (value == null) { - throw new IllegalStateException("Found invalid value in intermediate cache, for key " + key - + " during reduce phase execution on " + cacheManager.getAddress() + " for M/R task " - + taskId); - } - // and reduce it - VOut reduced = reducer.reduce(key, value.iterator()); - result.put(key, reduced); - log.tracef("For m/r task %s reduced %s to %s at %s ", taskId, key, reduced, cdl.getAddress()); - } - }); - - } finally { - if (trace) { - log.tracef("Reduce for task %s took %s milliseconds", reduceCommand.getTaskId(), - timeService.timeDuration(start, TimeUnit.MILLISECONDS)); - } - taskLifecycleService.onPostExecute(reducer); - } - } - } - - @SuppressWarnings("unchecked") - protected CollectableCollector map( - MapCombineCommand mcc) throws InterruptedException { - final Cache cache = cacheManager.getCache(mcc.getCacheName()); - Set keys = mcc.getKeys(); - int maxCSize = mcc.getMaxCollectorSize(); - final Mapper mapper = mcc.getMapper(); - final boolean inputKeysSpecified = keys != null && !keys.isEmpty(); - - // hook map function into lifecycle and execute it - MapReduceTaskLifecycleService taskLifecycleService = MapReduceTaskLifecycleService.getInstance(); - final CollectableCollector collector = new SynchronizedCollector( - new DefaultCollector(mcc, maxCSize)); - DataContainer dc = cache.getAdvancedCache().getDataContainer(); - log.tracef("For m/r task %s invoking %s with input keys %s", mcc.getTaskId(), mcc, keys); - long start = trace ? timeService.time() : 0; - try { - taskLifecycleService.onPreExecute(mapper, cache); - //User specified input taks keys, most likely a short list of input keys (<10^3), iterate serially - if (inputKeysSpecified) { - for (KIn key : keys) { - VIn value = cache.get(key); - if (value != null) { - mapper.map(key, value, collector); - } - } - } else { - // here we have to iterate all entries in memory, do it in parallel - dc.executeTask(new PrimaryOwnerFilter(cdl), new DataContainerTask() { - @Override - public void accept(KIn key , InternalCacheEntry v) { - VIn value = getValue(v); - if (value != null) { - mapper.map(key, value, collector); - } - } - }); - } - // in case we have stores, we have to process key/values from there as well - if (persistenceManager != null && !inputKeysSpecified) { - KeyFilter keyFilter = new CompositeKeyFilter(new PrimaryOwnerFilter(cdl), new CollectionKeyFilter(dc.keySet())); - persistenceManager.processOnAllStores(keyFilter, new MapReduceCacheLoaderTask(mapper, collector), - true, false); - } - } finally { - if (trace) { - log.tracef("Map phase for task %s took %s milliseconds", - mcc.getTaskId(), timeService.timeDuration(start, TimeUnit.MILLISECONDS)); - } - taskLifecycleService.onPostExecute(mapper); - } - return collector; - } - - @SuppressWarnings("unchecked") - protected Set mapAndCombine(final MapCombineCommand mcc) - throws Exception { - - final Cache cache = cacheManager.getCache(mcc.getCacheName()); - Set keys = mcc.getKeys(); - int maxCSize = mcc.getMaxCollectorSize(); - final Mapper mapper = mcc.getMapper(); - final boolean inputKeysSpecified = keys != null && !keys.isEmpty(); - // hook map function into lifecycle and execute it - MapReduceTaskLifecycleService taskLifecycleService = MapReduceTaskLifecycleService.getInstance(); - DataContainer dc = cache.getAdvancedCache().getDataContainer(); - log.tracef("For m/r task %s invoking %s with input keys %s", mcc.getTaskId(), mcc, mcc.getKeys()); - long start = trace ? timeService.time() : 0; - final Set intermediateKeys = new HashSet(); - try { - taskLifecycleService.onPreExecute(mapper, cache); - if (inputKeysSpecified) { - DefaultCollector c = new DefaultCollector(mcc, maxCSize); - for (KIn key : keys) { - VIn value = cache.get(key); - if (value != null) { - mapper.map(key, value, c); - } - } - combine(mcc, c); - Set s = migrateIntermediateKeysAndValues(mcc, c.collectedValues()); - intermediateKeys.addAll(s); - } else { - MapCombineTask task = new MapCombineTask(mcc, maxCSize); - dc.executeTask(new PrimaryOwnerFilter(cdl), task); - intermediateKeys.addAll(task.getMigratedIntermediateKeys()); - //the last chunk of remaining keys/values to migrate - Map> combinedValues = task.collectedValues(); - Set lastOne = migrateIntermediateKeysAndValues(mcc, combinedValues); - intermediateKeys.addAll(lastOne); - } - - // in case we have stores, we have to process key/values from there as well - if (persistenceManager != null && !inputKeysSpecified) { - KeyFilter keyFilter = new CompositeKeyFilter(new PrimaryOwnerFilter(cdl), - new CollectionKeyFilter(dc.keySet())); - - MapCombineTask task = new MapCombineTask(mcc, maxCSize); - persistenceManager.processOnAllStores(keyFilter, task, true, false); - intermediateKeys.addAll(task.getMigratedIntermediateKeys()); - //the last chunk of remaining keys/values to migrate - Map> combinedValues = task.collectedValues(); - Set lastOne = migrateIntermediateKeysAndValues(mcc, combinedValues); - intermediateKeys.addAll(lastOne); - } - } finally { - if (trace) { - log.tracef("Map phase for task %s took %s milliseconds", mcc.getTaskId(), - timeService.timeDuration(start, TimeUnit.MILLISECONDS)); - } - taskLifecycleService.onPostExecute(mapper); - } - return intermediateKeys; - } - - protected void combine(MapCombineCommand mcc, - CollectableCollector c) { - if (mcc.hasCombiner()) { - Reducer combiner = mcc.getCombiner(); - Cache cache = cacheManager.getCache(mcc.getCacheName()); - log.tracef("For m/r task %s invoking combiner %s at %s", mcc.getTaskId(), mcc, cdl.getAddress()); - MapReduceTaskLifecycleService taskLifecycleService = MapReduceTaskLifecycleService.getInstance(); - long start = trace ? timeService.time() : 0; - try { - taskLifecycleService.onPreExecute(combiner, cache); - for (Entry> e : c.collectedValues().entrySet()) { - List mapped = e.getValue(); - if (mapped.size() > 1) { - VOut reduced = combiner.reduce(e.getKey(), mapped.iterator()); - c.emitReduced(e.getKey(), reduced); - } - } - } finally { - if (trace) { - log.tracef("Combine for task %s took %s milliseconds", mcc.getTaskId(), - timeService.timeDuration(start, TimeUnit.MILLISECONDS)); - } - taskLifecycleService.onPostExecute(combiner); - } - } - } - - private Set migrateIntermediateKeysAndValues( - MapCombineCommand mcc, Map> collectedValues) { - - String taskId = mcc.getTaskId(); - String tmpCacheName = mcc.getIntermediateCacheName(); - Cache, DeltaList> tmpCache = cacheManager.getCache(tmpCacheName); - if (tmpCache == null) { - throw new IllegalStateException("Temporary cache for MapReduceTask " + taskId - + " named " + tmpCacheName + " not found on " + cdl.getAddress()); - } - - Set mapPhaseKeys = new HashSet(); - DistributionManager dm = tmpCache.getAdvancedCache().getDistributionManager(); - Map> keysToNodes = mapKeysToNodes(dm, taskId, collectedValues.keySet()); - long start = log.isTraceEnabled() ? timeService.time() : 0; - tmpCache = tmpCache.getAdvancedCache().withFlags(Flag.IGNORE_RETURN_VALUES); - try { - for (Entry> entry : keysToNodes.entrySet()) { - List keysHashedToAddress = entry.getValue(); - try { - log.tracef("For m/r task %s migrating intermediate keys %s to %s", taskId, keysHashedToAddress, entry.getKey()); - for (KOut key : keysHashedToAddress) { - List values = collectedValues.get(key); - int entryTransferCount = chunkSize; - for (int i = 0; i < values.size(); i += entryTransferCount) { - List chunk = values.subList(i, Math.min(values.size(), i + entryTransferCount)); - DeltaList delta = new DeltaList(chunk); - tmpCache.put(new IntermediateKey(taskId, key), delta); - } - mapPhaseKeys.add(key); - } - } catch (Exception e) { - throw new CacheException("Could not move intermediate keys/values for M/R task " + taskId, e); - } - } - } finally { - if (trace) { - log.tracef("Migrating keys for task %s took %s milliseconds (Migrated %s keys)", - mcc.getTaskId(), timeService.timeDuration(start, TimeUnit.MILLISECONDS), mapPhaseKeys.size()); - } - } - return mapPhaseKeys; - } - - @Override - public Map> mapKeysToNodes(DistributionManager dm, String taskId, - Collection keysToMap) { - Map> addressToKey = new HashMap>(); - for (T key : keysToMap) { - Address ownerOfKey = dm.getPrimaryLocation(new IntermediateKey(taskId, key)); - List keysAtNode = addressToKey.get(ownerOfKey); - if (keysAtNode == null) { - keysAtNode = new ArrayList(); - addressToKey.put(ownerOfKey, keysAtNode); - } - keysAtNode.add(key); - } - return addressToKey; - } - - protected Set filterLocalPrimaryOwner(Set nodeLocalKeys, DistributionManager dm) { - Set selectedKeys = new HashSet(); - for (KIn key : nodeLocalKeys) { - Address primaryLocation = dm != null ? dm.getPrimaryLocation(key) : cdl.getAddress(); - if (primaryLocation != null && primaryLocation.equals(cdl.getAddress())) { - selectedKeys.add(key); - } - } - return selectedKeys; - } - - private abstract class DataContainerTask implements BiConsumer> { - - @SuppressWarnings("unchecked") - protected V getValue(InternalCacheEntry entry){ - if (entry != null && !entry.isExpired(timeService.wallClockTime())) { - Object value = entry.getValue(); - if (value instanceof MarshalledValue) { - value = ((MarshalledValue) value).get(); - } - return (V)value; - } else { - return null; - } - } - } - - /** - * This is the parallel staggered map/combine algorithm. Threads from the default fork/join pool - * traverse container and store key/value pairs in parallel. As one of the threads hits the - * maxCollectorSize threshold, it takes the snapshot of the current state of the collector and - * invokes combine on it all while others threads continue to fill up collector up to the point - * where the threshold is reached again. The thread that broke the collector threshold invokes - * combine and the algorithm repeats. The benefit of staggered parallel map/combine is manyfold. - * First, we never exhaust working memory of a node as we batch map/combine execution all while - * traversal of key/value pairs is in progress. Second, such a staggered combine execution does - * not cause underlying transport to be completely saturated by intermediate cache put commands; - * intermediate key/value pairs of map/reduce algorithm are transferred across the cluster - * smoothly as parallel traversal of container's key/value pairs is progress. - * - */ - private final class MapCombineTask extends DataContainerTask implements AdvancedCacheLoader.CacheLoaderTask { - - private final MapCombineCommand mcc; - private final Set intermediateKeys; - private final int queueLimit; - private final BlockingQueue> queue; - - public MapCombineTask(MapCombineCommand mcc, int maxCollectorSize) throws Exception { - super(); - this.queueLimit = Runtime.getRuntime().availableProcessors() * 2; - this.queue = new ArrayBlockingQueue>(queueLimit + 1); - this.mcc = mcc; - this.intermediateKeys = Collections.synchronizedSet(new HashSet()); - //fill up queue with collectors - for (int i = 0; i < queueLimit; i++){ - queue.put(new DefaultCollector(mcc, maxCollectorSize)); - } - } - - @Override - public void accept(K key, InternalCacheEntry v) { - V value = getValue(v); - if (value != null) { - try { - executeMapWithCollector(key, value); - } catch (InterruptedException e) { - //reset signal - Thread.currentThread().interrupt(); - } - } - } - - @Override - public void processEntry(MarshalledEntry marshalledEntry, TaskContext taskContext) throws InterruptedException { - executeMapWithCollector(marshalledEntry.getKey(), getValue(marshalledEntry)); - } - - @Override - @SuppressWarnings("unchecked") - protected V getValue(InternalCacheEntry entry){ - if (entry != null) { - Object value = entry.getValue(); - if (value instanceof MarshalledValue) { - value = ((MarshalledValue) value).get(); - } - return (V)value; - } else { - return null; - } - } - - private Set getMigratedIntermediateKeys() { - return intermediateKeys; - } - - private Map> collectedValues() { - //combine all collectors from the queue into one - DefaultCollector finalCollector = new DefaultCollector(mcc, Integer.MAX_VALUE); - for (DefaultCollector collector : queue) { - if (!collector.isEmpty()) { - finalCollector.emit(collector.collectedValues()); - collector.reset(); - } - } - combine(mcc, finalCollector); - return finalCollector.collectedValues(); - } - - private void executeMapWithCollector(K key, V value) throws InterruptedException { - DefaultCollector c = null; - try { - // grab collector C from the bounded queue - c = queue.take(); - //invoke mapper with collector C - mcc.getMapper().map(key, value, c); - migrate(c); - } finally { - queue.put(c); - } - } - - private void migrate(final DefaultCollector c) { - // if overflow even after combine then migrate these keys/values - if (c.isOverflown()) { - Set migratedKeys = migrateIntermediateKeysAndValues(mcc, c.collectedValues()); - intermediateKeys.addAll(migratedKeys); - c.reset(); - } - } - - @SuppressWarnings("unchecked") - private V getValue(MarshalledEntry marshalledEntry) { - Object loadedValue = marshalledEntry.getValue(); - if (loadedValue instanceof MarshalledValue) { - return (V) ((MarshalledValue) loadedValue).get(); - } else { - return (V) loadedValue; - } - } - } - - private static final class IntermediateKeyFilter implements KeyFilter> { - - private final String taskId; - private final boolean acceptAll; - - public IntermediateKeyFilter(String taskId, boolean acceptAll) { - if (taskId == null || taskId.isEmpty()) { - throw new IllegalArgumentException("Invalid task Id " + taskId); - } - this.taskId = taskId; - this.acceptAll = acceptAll; - } - - @Override - public boolean accept(IntermediateKey key) { - if (acceptAll) { - return true; - } else { - if (key != null) { - return taskId.equals(key.getTaskId()); - } else { - return false; - } - } - } - } - - /** - * @author Sanne Grinovero (C) 2011 Red Hat Inc. - * @author Dan Berindei - * @author William Burns - * @author Vladimir Blagojevic - */ - private final class DefaultCollector implements CollectableCollector { - - private Map> store; - private final AtomicInteger emitCount; - private final int maxCollectorSize; - private MapCombineCommand mcc; - - public DefaultCollector(MapCombineCommand mcc, int maxCollectorSize) { - store = new HashMap>(1024, 0.75f); - emitCount = new AtomicInteger(); - this.maxCollectorSize = maxCollectorSize; - this.mcc = mcc; - } - - @Override - public void emit(KOut key, VOut value) { - List list = store.get(key); - if (list == null) { - list = new ArrayList(128); - store.put(key, list); - } - list.add(value); - emitCount.incrementAndGet(); - if (isOverflown() && mcc.hasCombiner()) { - combine(mcc, this); - } - } - - @Override - public void emitReduced(KOut key, VOut value) { - List list = store.get(key); - int prevSize = list.size(); - list.clear(); - list.add(value); - //we remove prevSize elements and replace it with one (the reduced value) - emitCount.addAndGet(-prevSize + 1); - } - - @Override - public Map> collectedValues() { - return store; - } - - public void reset(){ - store.clear(); - emitCount.set(0); - } - - public boolean isEmpty() { - return store.isEmpty(); - } - - public void emit(Map> combined) { - for (Entry> e : combined.entrySet()) { - KOut k = e.getKey(); - List values = e.getValue(); - for (VOut v : values) { - emit(k, v); - } - } - } - - public boolean isOverflown() { - return emitCount.get() > maxCollectorSize; - } - } - - private interface CollectableCollector extends Collector{ - Map> collectedValues(); - void emitReduced(K key, V value); - } - - private final class SynchronizedCollector implements CollectableCollector { - - private CollectableCollector delegate; - - public SynchronizedCollector(CollectableCollector delegate) { - this.delegate = delegate; - } - - @Override - public synchronized void emit(KOut key, VOut value) { - delegate.emit(key, value); - } - - @Override - public synchronized void emitReduced(KOut key, VOut value) { - delegate.emitReduced(key, value); - } - - @Override - public synchronized Map> collectedValues() { - return delegate.collectedValues(); - } - } - - private static class DeltaAwareList implements Iterable, DeltaAware { - - private final List list; - - public DeltaAwareList(List list) { - this.list = list; - } - - @Override - public Delta delta() { - return new DeltaList(list); - } - - @Override - public void commit() { - list.clear(); - } - - @Override - public Iterator iterator(){ - return list.iterator(); - } - - @Override - public String toString() { - return "DeltaAwareList(" + list.size() + ")" + String.valueOf(list); - } - } - - private static class DeltaList implements Delta { - - private final List deltas; - - public DeltaList(List list) { - deltas = new ArrayList(list); - } - - @SuppressWarnings("unchecked") - @Override - public DeltaAware merge(DeltaAware d) { - DeltaAwareList other = null; - if (d instanceof DeltaAwareList) { - other = (DeltaAwareList) d; - other.list.addAll(deltas); - } else { - other = new DeltaAwareList(deltas); - } - return other; - } - } - - @SuppressWarnings("rawtypes") - public static class DeltaListExternalizer extends AbstractExternalizer { - - private static final long serialVersionUID = 5859147782602054109L; - - @Override - public void writeObject(ObjectOutput output, DeltaList list) throws IOException { - output.writeObject(list.deltas); - } - - @Override - @SuppressWarnings("unchecked") - public DeltaList readObject(ObjectInput input) throws IOException, ClassNotFoundException { - return new DeltaList((List) input.readObject()); - } - - @Override - public Integer getId() { - return Ids.DELTA_MAPREDUCE_LIST_ID; - } - - @Override - @SuppressWarnings("unchecked") - public Set> getTypeClasses() { - return Util.>asSet(DeltaList.class); - } - } - - @SuppressWarnings("rawtypes") - public static class DeltaAwareListExternalizer extends AbstractExternalizer { - - private static final long serialVersionUID = -8956663669844107351L; - - @Override - public void writeObject(ObjectOutput output, DeltaAwareList deltaAwareList) throws IOException { - output.writeObject(deltaAwareList.list); - } - - @Override - @SuppressWarnings("unchecked") - public DeltaAwareList readObject(ObjectInput input) throws IOException, ClassNotFoundException { - return new DeltaAwareList((List) input.readObject()); - } - - @Override - public Integer getId() { - return Ids.DELTA_AWARE_MAPREDUCE_LIST_ID; - } - - @Override - @SuppressWarnings("unchecked") - public Set> getTypeClasses() { - return Util.>asSet(DeltaAwareList.class); - } - } - - /** - * IntermediateCompositeKey - */ - public static final class IntermediateKey implements Serializable { - - /** The serialVersionUID */ - private static final long serialVersionUID = 4434717760740027918L; - - private final String taskId; - private final V key; - - public IntermediateKey(String taskId, V key) { - this.taskId = taskId; - this.key = key; - } - - public String getTaskId() { - return taskId; - } - - public V getKey(){ - return key; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((key == null) ? 0 : key.hashCode()); - result = prime * result + ((taskId == null) ? 0 : taskId.hashCode()); - return result; - } - - @SuppressWarnings("unchecked") - @Override - public boolean equals(Object obj) { - if (obj == null) { - return false; - } - if (!(obj instanceof IntermediateKey)) { - return false; - } - IntermediateKey other = (IntermediateKey) obj; - if (key == null) { - if (other.key != null) { - return false; - } - } else if (!key.equals(other.key)) { - return false; - } - if (taskId == null) { - if (other.taskId != null) { - return false; - } - } else if (!taskId.equals(other.taskId)) { - return false; - } - return true; - } - - @Override - public String toString() { - return "IntermediateCompositeKey [taskId=" + taskId + ", key=" + key + "]"; - } - } -} diff --git a/core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceTask.java b/core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceTask.java deleted file mode 100644 index 7d48037fdd8c..000000000000 --- a/core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceTask.java +++ /dev/null @@ -1,1229 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.AdvancedCache; -import org.infinispan.Cache; -import org.infinispan.CacheSet; -import org.infinispan.commands.CancelCommand; -import org.infinispan.commands.CancellationService; -import org.infinispan.commands.CommandsFactory; -import org.infinispan.commands.CreateCacheCommand; -import org.infinispan.commands.read.MapCombineCommand; -import org.infinispan.commands.read.ReduceCommand; -import org.infinispan.commons.CacheException; -import org.infinispan.commons.marshall.Marshaller; -import org.infinispan.commons.marshall.StreamingMarshaller; -import org.infinispan.commons.util.Util; -import org.infinispan.commons.util.concurrent.NotifyingFutureImpl; -import org.infinispan.distexec.mapreduce.MapReduceManagerImpl.IntermediateKey; -import org.infinispan.distexec.mapreduce.spi.MapReduceTaskLifecycleService; -import org.infinispan.distribution.DistributionManager; -import org.infinispan.factories.ComponentRegistry; -import org.infinispan.interceptors.base.CommandInterceptor; -import org.infinispan.interceptors.locking.ClusteringDependentLogic; -import org.infinispan.lifecycle.ComponentStatus; -import org.infinispan.manager.EmbeddedCacheManager; -import org.infinispan.remoting.responses.Response; -import org.infinispan.remoting.responses.SuccessfulResponse; -import org.infinispan.remoting.rpc.RpcManager; -import org.infinispan.remoting.rpc.RpcOptionsBuilder; -import org.infinispan.remoting.transport.Address; -import org.infinispan.security.AuthorizationManager; -import org.infinispan.security.AuthorizationPermission; -import org.infinispan.util.logging.Log; -import org.infinispan.util.logging.LogFactory; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.Callable; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import static org.infinispan.factories.KnownComponentNames.CACHE_MARSHALLER; - -/** - * MapReduceTask is a distributed task allowing a large scale computation to be transparently - * parallelized across Infinispan cluster nodes. - *

- * - * Users should instantiate MapReduceTask with a reference to a cache whose data is used as input for this - * task. Infinispan execution environment will migrate and execute instances of provided {@link Mapper} - * and {@link Reducer} seamlessly across Infinispan nodes. - *

- * - * Unless otherwise specified using {@link MapReduceTask#onKeys(Object...)} filter all available - * key/value pairs of a specified cache will be used as input data for this task. - * - * For example, MapReduceTask that counts number of word occurrences in a particular cache where - * keys and values are String instances could be written as follows: - * - * 

- * MapReduceTask<String, String, String, Integer> task = new MapReduceTask<String, String, String, Integer>(cache);
- * task.mappedWith(new WordCountMapper()).reducedWith(new WordCountReducer());
- * Map<String, Integer> results = task.execute();
- *
- * - * The final result is a map where key is a word and value is a word count for that particular word. - *

- * - * Accompanying {@link Mapper} and {@link Reducer} are defined as follows: - * - * 

- *    private static class WordCountMapper implements Mapper<String, String, String,Integer> {
- *
- *     public void map(String key, String value, Collector<String, Integer> collector) {
- *        StringTokenizer tokens = new StringTokenizer(value);
- *       while (tokens.hasMoreElements()) {
- *           String s = (String) tokens.nextElement();
- *           collector.emit(s, 1);
- *        }
- *     }
- *  }
- *
- *   private static class WordCountReducer implements Reducer<String, Integer> {
- *
- *      public Integer reduce(String key, Iterator<Integer> iter) {
- *         int sum = 0;
- *         while (iter.hasNext()) {
- *            Integer i = (Integer) iter.next();
- *            sum += i;
- *        }
- *         return sum;
- *      }
- *   }
- *
- * - *

- * - * Finally, as of Infinispan 5.2 release, MapReduceTask can also specify a Combiner function. The Combiner - * is executed on each node after the Mapper and before the global reduce phase. The Combiner receives input from - * the Mapper's output and the output from the Combiner is then sent to the reducers. It is useful to think - * of the Combiner as a node local reduce phase before global reduce phase is executed. - *

- * - * Combiners are especially useful when reduce function is both commutative and associative! In such cases - * we can use the Reducer itself as the Combiner; all one needs to do is to specify the Combiner: - * 

- * MapReduceTask<String, String, String, Integer> task = new MapReduceTask<String, String, String, Integer>(cache);
- * task.mappedWith(new WordCountMapper()).reducedWith(new WordCountReducer()).combineWith(new WordCountReducer());
- * Map<String, Integer> results = task.execute();
- *
- * - * Note that {@link Mapper} and {@link Reducer} should not be specified as inner classes. Inner classes - * declared in non-static contexts contain implicit non-transient references to enclosing class instances, - * serializing such an inner class instance will result in serialization of its associated outer class instance as well. - * - *

- * - * If you are not familiar with concept of map reduce distributed execution model - * start with Google's MapReduce research paper. - * - * - * @author Manik Surtani - * @author Vladimir Blagojevic - * @author Sanne Grinovero - * - * @since 5.0 - * @deprecated Map reduce has been replaced by streams. This can be invoked by cache.entrySet().stream().... - * @see Cache#entrySet() - * @see CacheSet#stream() - */ -public class MapReduceTask { - - private static final Log log = LogFactory.getLog(MapReduceTask.class); - public static final String DEFAULT_TMP_CACHE_CONFIGURATION_NAME= "__tmpMapReduce"; - - protected Mapper mapper; - protected Reducer reducer; - protected Reducer combiner; - protected final boolean distributeReducePhase; - protected boolean useIntermediateSharedCache; - - protected final Collection keys; - protected final AdvancedCache cache; - protected final Marshaller marshaller; - protected final MapReduceManager mapReduceManager; - protected final CancellationService cancellationService; - protected final List cancellableTasks; - protected final UUID taskId; - protected final ClusteringDependentLogic clusteringDependentLogic; - protected final boolean isLocalOnly; - protected RpcOptionsBuilder rpcOptionsBuilder; - protected String customIntermediateCacheName; - protected String intermediateCacheConfigurationName = DEFAULT_TMP_CACHE_CONFIGURATION_NAME; - private static final int MAX_COLLECTOR_SIZE = 1000; - - /** - * Create a new MapReduceTask given a master cache node. All distributed task executions will be - * initiated from this cache node. This task will by default only use distributed map phase while - * reduction will be executed on task originating Infinispan node. - *

- * - * Large and data intensive tasks whose reduction phase would exceed working memory of one - * Infinispan node should use distributed reduce phase - * - * @param masterCacheNode - * cache node initiating map reduce task - */ - public MapReduceTask(Cache masterCacheNode) { - this(masterCacheNode, false, false); - } - - /** - * Create a new MapReduceTask given a master cache node. All distributed task executions will be - * initiated from this cache node. - * - * @param masterCacheNode - * cache node initiating map reduce task - * @param distributeReducePhase - * if true this task will use distributed reduce phase execution - * - */ - public MapReduceTask(Cache masterCacheNode, boolean distributeReducePhase) { - this(masterCacheNode, distributeReducePhase, true); - } - - - /** - * Create a new MapReduceTask given a master cache node. All distributed task executions will be - * initiated from this cache node. - * - * @param masterCacheNode - * cache node initiating map reduce task - * @param distributeReducePhase - * if true this task will use distributed reduce phase execution - * @param useIntermediateSharedCache - * if true this tasks will share intermediate value cache with other executing - * MapReduceTasks on the grid. Otherwise, if false, this task will use its own - * dedicated cache for intermediate values - * - */ - public MapReduceTask(Cache masterCacheNode, boolean distributeReducePhase, boolean useIntermediateSharedCache) { - if (masterCacheNode == null) - throw new IllegalArgumentException("Can not use null cache for MapReduceTask"); - ensureAccessPermissions(masterCacheNode.getAdvancedCache()); - ensureProperCacheState(masterCacheNode.getAdvancedCache()); - ensureFullCache(masterCacheNode.getAdvancedCache()); - this.cache = masterCacheNode.getAdvancedCache(); - this.keys = new LinkedList(); - ComponentRegistry componentRegistry = SecurityActions.getCacheComponentRegistry(cache); - this.marshaller = componentRegistry.getComponent(StreamingMarshaller.class, CACHE_MARSHALLER); - this.mapReduceManager = componentRegistry.getComponent(MapReduceManager.class); - this.cancellationService = componentRegistry.getComponent(CancellationService.class); - this.taskId = UUID.randomUUID(); - if (useIntermediateSharedCache) { - this.customIntermediateCacheName = DEFAULT_TMP_CACHE_CONFIGURATION_NAME; - } else { - this.customIntermediateCacheName = taskId.toString(); - } - this.distributeReducePhase = distributeReducePhase; - this.useIntermediateSharedCache = useIntermediateSharedCache; - this.cancellableTasks = Collections.synchronizedList(new ArrayList()); - this.clusteringDependentLogic = componentRegistry.getComponent(ClusteringDependentLogic.class); - this.isLocalOnly = SecurityActions.getCacheRpcManager(cache) == null; - this.rpcOptionsBuilder = isLocalOnly ? null : new RpcOptionsBuilder(SecurityActions.getCacheRpcManager(cache).getDefaultRpcOptions(true)); - if (!isLocalOnly) { - this.rpcOptionsBuilder.timeout(0, TimeUnit.MILLISECONDS); - } - } - - /** - * Rather than use all available keys as input onKeys allows users to specify a - * subset of keys as input to this task - * - * @param input - * input keys for this task - * @return this task - */ - public MapReduceTask onKeys(KIn... input) { - Collections.addAll(keys, input); - return this; - } - - /** - * Specifies Mapper to use for this MapReduceTask - *

- * Note that {@link Mapper} should not be specified as inner class. Inner classes declared in - * non-static contexts contain implicit non-transient references to enclosing class instances, - * serializing such an inner class instance will result in serialization of its associated outer - * class instance as well. - * - * @param mapper used to execute map phase of MapReduceTask - * @return this MapReduceTask itself - */ - public MapReduceTask mappedWith(Mapper mapper) { - if (mapper == null) - throw new IllegalArgumentException("A valid reference of Mapper is needed"); - this.mapper = mapper; - return this; - } - - /** - * Specifies Reducer to use for this MapReduceTask - * - *

- * Note that {@link Reducer} should not be specified as inner class. Inner classes declared in - * non-static contexts contain implicit non-transient references to enclosing class instances, - * serializing such an inner class instance will result in serialization of its associated outer - * class instance as well. - * - * @param reducer used to reduce results of map phase - * @return this MapReduceTask itself - */ - public MapReduceTask reducedWith(Reducer reducer) { - if (reducer == null) - throw new IllegalArgumentException("A valid reference of Reducer is needed"); - this.reducer = reducer; - return this; - } - - /** - * Specifies Combiner to use for this MapReduceTask - * - *

- * Note that {@link Reducer} should not be specified as inner class. Inner classes declared in - * non-static contexts contain implicit non-transient references to enclosing class instances, - * serializing such an inner class instance will result in serialization of its associated outer - * class instance as well. - * - * @param combiner used to immediately combine results of map phase before reduce phase is invoked - * @return this MapReduceTask itself - * @since 5.2 - */ - public MapReduceTask combinedWith(Reducer combiner) { - if (combiner == null) - throw new IllegalArgumentException("A valid reference of Reducer/Combiner is needed"); - this.combiner = combiner; - return this; - } - - /** - * See {@link #timeout(TimeUnit)}. - * - * Note: the timeout value will be converted to milliseconds and a value less or equal than zero means wait forever. - * The default timeout for this task is 0. The task will wait indefinitely for its completion. - * - * @param timeout - * @param unit - * @return this MapReduceTask itself - */ - public final MapReduceTask timeout(long timeout, TimeUnit unit) { - rpcOptionsBuilder.timeout(timeout, unit); - return this; - } - - /** - * @return the timeout value in {@link TimeUnit} to wait for the remote map/reduce task to finish. The default - * timeout is 0, the task will wait indefinitely for its completion. - */ - public final long timeout(TimeUnit outputTimeUnit) { - return rpcOptionsBuilder.timeout(outputTimeUnit); - } - - /** - * - * Allows this MapReduceTask to use specific intermediate custom defined cache for storage of - * intermediate > key/values pairs. Intermediate cache is used to store output - * of map phase and it is not shared with other M/R tasks. Upon completion of M/R task this intermediate - * cache is destroyed. - * - * @param cacheConfigurationName - * name of the cache configuration to use for the intermediate cache - * @return this MapReduceTask iteself - * @since 7.0 - */ - public MapReduceTask usingIntermediateCache(String cacheConfigurationName) { - if (cacheConfigurationName == null || cacheConfigurationName.isEmpty()) { - throw new IllegalArgumentException("Invalid configuration name " + cacheConfigurationName - + ", cacheConfigurationName cannot be null or empty"); - } - this.intermediateCacheConfigurationName = cacheConfigurationName; - this.useIntermediateSharedCache = false; - return this; - } - - /** - * Allows this MapReduceTask to use a specific shared intermediate cache for storage of - * intermediate > key/values pairs. Intermediate shared cache is used to store - * output of map phase and it is shared with other M/R tasks that also specify a shared - * intermediate cache with the same name. - * - * @param cacheName - * name of the custom cache - * @return this MapReduceTask iteself - * @since 7.0 - */ - public MapReduceTask usingSharedIntermediateCache(String cacheName) { - if (cacheName == null || cacheName.isEmpty()) { - throw new IllegalArgumentException("Invalid cache name" + cacheName + ", cache name cannot be null or empty"); - } - this.customIntermediateCacheName = cacheName; - this.useIntermediateSharedCache = true; - return this; - } - - /** - * Allows this MapReduceTask to use a specific shared intermediate cache for storage of - * intermediate > key/values pairs. Intermediate shared cache is used to store - * output of map phase and it is shared with other M/R tasks that also specify a shared - * intermediate cache with the same name. - *

- * Rather than using MapReduceTask default configuration for intermediate cache this method - * allows clients to specify custom shared cache configuration. - * - * - * @param cacheName - * name of the custom cache - * @param cacheConfigurationName - * name of the cache configuration to use for the intermediate cache - * @return this MapReduceTask iteself - * @since 7.0 - */ - public MapReduceTask usingSharedIntermediateCache(String cacheName, String cacheConfigurationName) { - if (cacheConfigurationName == null || cacheConfigurationName.isEmpty()) { - throw new IllegalArgumentException("Invalid configuration name " + cacheConfigurationName - + ", cacheConfigurationName cannot be null or empty"); - } - if (cacheName == null || cacheName.isEmpty()) { - throw new IllegalArgumentException("Invalid cache name" + cacheName + ", cache name cannot be null or empty"); - } - this.customIntermediateCacheName = cacheName; - this.intermediateCacheConfigurationName = cacheConfigurationName; - this.useIntermediateSharedCache = true; - return this; - } - - /** - * Executes this task across Infinispan cluster nodes. - * - * @return a Map where each key is an output key and value is reduced value for that output key - */ - public Map execute() throws CacheException { - return executeHelper(null); - } - - /** - * Executes this task and stores results in the provided results cache. The results can be - * obtained once the execute method completes i.e., execute method is synchronous. - * - * This variant of execute method minimizes the possibility of the master JVM node exceeding - * its allowed maximum heap, especially if objects that are results of the reduce phase have a - * large memory footprint and/or multiple MapReduceTasks are executed concurrently on the master - * task node. - * - * @param resultsCache - * application provided results cache - * @throws CacheException - * - * @since 7.0 - */ - public void execute(Cache resultsCache) throws CacheException { - executeHelper(resultsCache.getName()); - } - - /** - * Executes this task and stores results in the provided results cache. The results can be - * obtained once the execute method completes i.e., execute method is synchronous. - * - * This variant of execute method minimizes the possibility of the master JVM node exceeding its - * allowed maximum heap, especially if objects that are results of the reduce phase have a large - * memory footprint and/or multiple MapReduceTasks are executed concurrently on the master task - * node. - * - * @param resultsCache - * application provided results cache represented by its name - * @throws CacheException - * - * @since 7.0 - */ - public void execute(String resultsCache) throws CacheException { - if (resultsCache == null || resultsCache.isEmpty()) { - throw new IllegalArgumentException("Results cache can not be " + resultsCache); - } - executeHelper(resultsCache); - } - - protected Map executeHelper(String resultCache) throws NullPointerException, MapReduceException { - ensureAccessPermissions(cache); - if (mapper == null) - throw new NullPointerException("A valid reference of Mapper is not set " + mapper); - - if (reducer == null) - throw new NullPointerException("A valid reference of Reducer is not set " + reducer); - - Map result = null; - if(!isLocalOnly && distributeReducePhase()){ - // init and create tmp caches - executeTaskInit(getIntermediateCacheName()); - Set allMapPhasesResponses = null; - try { - // map - allMapPhasesResponses = executeMapPhase(); - - // reduce - result = executeReducePhase(resultCache, allMapPhasesResponses, useIntermediateSharedCache()); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new MapReduceException(ie); - } - finally { - // cleanup tmp caches across cluster - EmbeddedCacheManager cm = cache.getCacheManager(); - String intermediateCache = getIntermediateCacheName(); - if (useIntermediatePerTaskCache()) { - cm.removeCache(intermediateCache); - } else { - //let's make sure shared cache is not destroyed and that we have keys to remove - Cache sharedTmpCache = cm.getCache(intermediateCache); - if (sharedTmpCache != null && allMapPhasesResponses != null) { - for (KOut k : allMapPhasesResponses) { - sharedTmpCache.removeAsync(new IntermediateKey(taskId.toString(), k)); - } - } - } - } - } else { - try { - if (resultCache == null || resultCache.isEmpty()) { - result = new HashMap(); - executeMapPhaseWithLocalReduction(result); - } else { - EmbeddedCacheManager cm = cache.getCacheManager(); - Cache c = cm.getCache(resultCache); - executeMapPhaseWithLocalReduction(c); - } - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new MapReduceException(ie); - } - } - return result; - } - - protected String getIntermediateCacheName() { - return customIntermediateCacheName; - } - - protected boolean distributeReducePhase(){ - return distributeReducePhase; - } - - protected boolean useIntermediateSharedCache() { - return useIntermediateSharedCache; - } - - protected boolean useIntermediatePerTaskCache() { - return !useIntermediateSharedCache(); - } - - - protected void executeTaskInit(String tmpCacheName) throws MapReduceException { - RpcManager rpc = cache.getRpcManager(); - CommandsFactory factory = cache.getComponentRegistry().getComponent(CommandsFactory.class); - - //first create tmp caches on all nodes - final CreateCacheCommand ccc = factory.buildCreateCacheCommand(tmpCacheName, - intermediateCacheConfigurationName, - rpc.getMembers().size()); - log.debugf("Invoking %s across members %s ", ccc, cache.getRpcManager().getMembers()); - - // invoke remotely - CompletableFuture> remoteFuture = rpc.invokeRemotelyAsync( - cache.getRpcManager().getMembers(), ccc, rpcOptionsBuilder.build()); - - // invoke locally - try { - ccc.init(cache.getCacheManager()); - ccc.perform(null); - } catch (Throwable e) { - throw new MapReduceException(e); - } - - // process remote responses - try { - Map map = remoteFuture.get(); - for (Entry e : map.entrySet()) { - if (!e.getValue().isSuccessful()) { - throw new MapReduceException( - "Could not initialize tmp cache " + tmpCacheName + " at " + e.getKey() + " for " + - this); - } - } - } catch (InterruptedException | ExecutionException e) { - throw new MapReduceException( - "Could not initialize temporary caches for MapReduce task on remote nodes ", e); - } - } - - protected Set executeMapPhase() throws MapReduceException, InterruptedException { - RpcManager rpc = cache.getRpcManager(); - MapCombineCommand cmd = null; - Set mapPhasesResult = new HashSet(); - List>> futures = new ArrayList>>(); - if (inputTaskKeysEmpty()) { - for (Address target : rpc.getMembers()) { - if (target.equals(rpc.getAddress())) { - cmd = buildMapCombineCommand(taskId.toString(), clone(mapper), clone(combiner), - getIntermediateCacheName(), null, true, useIntermediateSharedCache()); - } else { - cmd = buildMapCombineCommand(taskId.toString(), mapper, combiner, getIntermediateCacheName(), null, - true, useIntermediateSharedCache()); - } - MapTaskPart> part = createTaskMapPart(cmd, target, true); - part.execute(); - futures.add(part); - } - } else { - Map> keysToNodes = mapKeysToNodes(keys); - for (Entry> e : keysToNodes.entrySet()) { - Address address = e.getKey(); - Collection keys = e.getValue(); - if (address.equals(rpc.getAddress())) { - cmd = buildMapCombineCommand(taskId.toString(), clone(mapper), clone(combiner), - getIntermediateCacheName(), keys, true, useIntermediateSharedCache()); - } else { - cmd = buildMapCombineCommand(taskId.toString(), mapper, combiner, getIntermediateCacheName(), keys, - true, useIntermediateSharedCache()); - } - MapTaskPart> part = createTaskMapPart(cmd, address, true); - part.execute(); - futures.add(part); - } - } - try { - for (MapTaskPart> mapTaskPart : futures) { - Set result = null; - try { - result = mapTaskPart.get(); - } catch (ExecutionException ee) { - throw new MapReduceException("Map phase failed ", ee.getCause()); - } - mapPhasesResult.addAll(result); - } - } finally { - cancellableTasks.clear(); - } - return mapPhasesResult; - } - - protected void executeMapPhaseWithLocalReduction(Map reducedResult) throws MapReduceException, InterruptedException { - RpcManager rpc = SecurityActions.getCacheRpcManager(cache); - MapCombineCommand cmd = null; - Map> mapPhasesResult = new HashMap>(); - List>>> futures = new ArrayList>>>(); - Address localAddress = clusteringDependentLogic.getAddress(); - if (inputTaskKeysEmpty()) { - List

targets; - if (isLocalOnly) { - targets = Collections.singletonList(localAddress); - } else { - targets = rpc.getMembers(); - } - for (Address target : targets) { - if (target.equals(localAddress)) { - cmd = buildMapCombineCommand(taskId.toString(), clone(mapper), clone(combiner), - getIntermediateCacheName(), null, false, false); - } else { - cmd = buildMapCombineCommand(taskId.toString(), mapper, combiner, getIntermediateCacheName(), null, - false, false); - } - MapTaskPart>> part = createTaskMapPart(cmd, target, false); - part.execute(); - futures.add(part); - } - } else { - Map> keysToNodes = mapKeysToNodes(keys); - for (Entry> e : keysToNodes.entrySet()) { - Address address = e.getKey(); - Collection keys = e.getValue(); - if (address.equals(localAddress)) { - cmd = buildMapCombineCommand(taskId.toString(), clone(mapper), clone(combiner), - getIntermediateCacheName(), keys, false, false); - } else { - cmd = buildMapCombineCommand(taskId.toString(), mapper, combiner, getIntermediateCacheName(), keys, - false, false); - } - MapTaskPart>> part = createTaskMapPart(cmd, address, false); - part.execute(); - futures.add(part); - } - } - try { - for (MapTaskPart>> mapTaskPart : futures) { - Map> result = null; - try { - result = mapTaskPart.get(); - } catch (ExecutionException ee) { - throw new MapReduceException("Map phase failed ", ee.getCause()); - } - mergeResponse(mapPhasesResult, result); - } - } finally { - cancellableTasks.clear(); - } - - // hook into lifecycle - MapReduceTaskLifecycleService taskLifecycleService = MapReduceTaskLifecycleService - .getInstance(); - log.tracef("For m/r task %s invoking %s locally", taskId, reducer); - try { - taskLifecycleService.onPreExecute(reducer, cache); - for (Entry> e : mapPhasesResult.entrySet()) { - // TODO in parallel with futures - reducedResult.put(e.getKey(), reducer.reduce(e.getKey(), e.getValue().iterator())); - } - } finally { - taskLifecycleService.onPostExecute(reducer); - } - } - - protected MapTaskPart createTaskMapPart(MapCombineCommand cmd, - Address target, boolean distributedReduce) { - MapTaskPart mapTaskPart = new MapTaskPart(target, cmd, distributedReduce); - cancellableTasks.add(mapTaskPart); - return mapTaskPart; - } - - protected Map executeReducePhase(String resultCache, Set allMapPhasesResponses, - boolean useIntermediateSharedCache) throws MapReduceException, InterruptedException { - RpcManager rpc = cache.getRpcManager(); - String destCache = getIntermediateCacheName(); - - Cache dstCache = cache.getCacheManager().getCache(destCache); - Map> keysToNodes = mapKeysToNodes(dstCache.getAdvancedCache() - .getDistributionManager(), allMapPhasesResponses, useIntermediateSharedCache); - Map reduceResult = new HashMap(); - List>> reduceTasks = new ArrayList>>(); - ReduceCommand reduceCommand = null; - for (Entry> e : keysToNodes.entrySet()) { - Address address = e.getKey(); - Collection keys = e.getValue(); - if (address.equals(rpc.getAddress())) { - reduceCommand = buildReduceCommand(resultCache, taskId.toString(), destCache, clone(reducer), keys, - useIntermediateSharedCache); - } else { - reduceCommand = buildReduceCommand(resultCache, taskId.toString(), destCache, reducer, keys, - useIntermediateSharedCache); - } - ReduceTaskPart> part = createReducePart(reduceCommand, address, destCache); - part.execute(); - reduceTasks.add(part); - } - try { - for (ReduceTaskPart> reduceTaskPart : reduceTasks) { - Map result = null; - try { - result = reduceTaskPart.get(); - } catch (ExecutionException ee) { - throw new MapReduceException("Reduce phase failed", ee.getCause()); - } - reduceResult.putAll(result); - } - } finally { - cancellableTasks.clear(); - } - return reduceResult; - } - - protected ReduceTaskPart createReducePart(ReduceCommand cmd, Address target, - String destCacheName) { - ReduceTaskPart part = new ReduceTaskPart(target, cmd, destCacheName); - cancellableTasks.add(part); - return part; - } - - private void mergeResponse(Map> result, Map> m) { - for (Entry> entry : m.entrySet()) { - synchronized (result) { - List list = result.get(entry.getKey()); - if (list != null) { - list.addAll(entry.getValue()); - } else { - list = new ArrayList(); - list.addAll(entry.getValue()); - } - result.put(entry.getKey(), list); - } - } - } - - private MapCombineCommand buildMapCombineCommand( - String taskId, Mapper m, Reducer r, String intermediateCacheName, - Collection keys, boolean reducePhaseDistributed, boolean useIntermediateSharedCache){ - ComponentRegistry registry = SecurityActions.getCacheComponentRegistry(cache); - CommandsFactory factory = registry.getComponent(CommandsFactory.class); - MapCombineCommand c = factory.buildMapCombineCommand(taskId, m, r, keys); - c.setReducePhaseDistributed(reducePhaseDistributed); - c.setUseIntermediateSharedCache(useIntermediateSharedCache); - c.setIntermediateCacheName(intermediateCacheName); - c.setMaxCollectorSize(MAX_COLLECTOR_SIZE); - return c; - } - - private ReduceCommand buildReduceCommand(String resultCacheName, String taskId, String destinationCache, - Reducer r, Collection keys, boolean useIntermediateSharedCache) { - ComponentRegistry registry = cache.getComponentRegistry(); - CommandsFactory factory = registry.getComponent(CommandsFactory.class); - ReduceCommand reduceCommand = factory.buildReduceCommand(taskId, destinationCache, r, keys); - reduceCommand.setUseIntermediateSharedCache(useIntermediateSharedCache); - reduceCommand.setResultCacheName(resultCacheName); - return reduceCommand; - } - - private CancelCommand buildCancelCommand(CancellableTaskPart taskPart){ - ComponentRegistry registry = cache.getComponentRegistry(); - CommandsFactory factory = registry.getComponent(CommandsFactory.class); - return factory.buildCancelCommandCommand(taskPart.getUUID()); - } - - /** - * Executes this task across Infinispan cluster nodes asynchronously. - * - * @return a Future wrapping a Map where each key is an output key and value is reduced value for - * that output key - */ - public Future> executeAsynchronously() { - final MapReduceTaskFuture> result = new MapReduceTaskFuture>(); - ExecutorService executor = mapReduceManager.getExecutorService(); - Future> returnValue = executor.submit(new Callable>() { - @Override - public Map call() throws Exception { - try { - Map retval = execute(); - try { - result.notifyDone(retval); - log.trace("Finished notifying"); - } catch (Throwable e) { - log.trace("Exception while notifying the future", e); - } - return retval; - } catch (Exception e) { - try { - result.notifyException(e); - log.trace("Finished notifying exception"); - } catch (Throwable e2) { - log.trace("Exception while notifying the future", e2); - } - throw e; - } - } - }); - result.setFuture(returnValue); - return result; - } - - /** - * Executes this task across Infinispan cluster but the final result is collated using specified - * {@link Collator} - * - * @param collator - * a Collator to use - * - * @return collated result - */ - public R execute(Collator collator) { - Map execute = execute(); - return collator.collate(execute); - } - - /** - * Executes this task asynchronously across Infinispan cluster; final result is collated using - * specified {@link Collator} and wrapped by Future - * - * @param collator - * a Collator to use - * - * @return collated result - */ - public Future executeAsynchronously(final Collator collator) { - final MapReduceTaskFuture result = new MapReduceTaskFuture(); - ExecutorService executor = mapReduceManager.getExecutorService(); - Future returnValue = executor.submit(new Callable() { - @Override - public R call() throws Exception { - try { - R retval = execute(collator); - try { - result.notifyDone(retval); - log.trace("Finished notifying"); - } catch (Throwable e) { - log.trace("Exception while notifying the future", e); - } - return retval; - } catch (Exception e) { - try { - result.notifyException(e); - log.trace("Finished notifying"); - } catch (Throwable e2) { - log.trace("Exception while notifying the future", e2); - } - throw e; - } - } - }); - result.setFuture(returnValue); - return result; - } - - protected Map> mapKeysToNodes(DistributionManager dm, Collection keysToMap, boolean useIntermediateCompositeKey) { - if (isLocalOnly) { - return Collections.singletonMap(clusteringDependentLogic.getAddress(), keysToMap); - } else { - return mapReduceManager.mapKeysToNodes(dm, taskId.toString(), keysToMap); - } - } - - protected Map> mapKeysToNodes(Collection keysToMap, boolean useIntermediateCompositeKey) { - return mapKeysToNodes(cache.getDistributionManager(), keysToMap, useIntermediateCompositeKey); - } - - protected Map> mapKeysToNodes(Collection keysToMap) { - return mapKeysToNodes(keysToMap, false); - } - - protected Mapper clone(Mapper mapper){ - return Util.cloneWithMarshaller(marshaller, mapper); - } - - protected Reducer clone(Reducer reducer){ - return Util.cloneWithMarshaller(marshaller, reducer); - } - - private void ensureAccessPermissions(final AdvancedCache cache) { - AuthorizationManager authorizationManager = SecurityActions.getCacheAuthorizationManager(cache); - if (authorizationManager != null) { - authorizationManager.checkPermission(AuthorizationPermission.EXEC); - } - } - - private void ensureProperCacheState(AdvancedCache cache) throws NullPointerException, - IllegalStateException { - if (cache.getStatus() != ComponentStatus.RUNNING) - throw log.invalidCacheState(cache.getStatus().toString()); - - if (SecurityActions.getCacheRpcManager(cache) != null && SecurityActions.getCacheDistributionManager(cache) == null) { - throw log.requireDistOrReplCache(cache.getCacheConfiguration().clustering().cacheModeString()); - } - } - - private void ensureFullCache(AdvancedCache advancedCache) { - List interceptorChain = SecurityActions.getInterceptorChain(advancedCache); - if (interceptorChain == null || interceptorChain.isEmpty()) { - throw log.mapReduceNotSupported(); - } - } - - protected boolean inputTaskKeysEmpty() { - return keys == null || keys.isEmpty(); - } - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((taskId == null) ? 0 : taskId.hashCode()); - return result; - } - - @SuppressWarnings("rawtypes") - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (!(obj instanceof MapReduceTask)) { - return false; - } - MapReduceTask other = (MapReduceTask) obj; - if (taskId == null) { - if (other.taskId != null) { - return false; - } - } else if (!taskId.equals(other.taskId)) { - return false; - } - return true; - } - - @Override - public String toString() { - return "MapReduceTask [mapper=" + mapper + ", reducer=" + reducer + ", combiner=" + combiner - + ", keys=" + keys + ", taskId=" + taskId + "]"; - } - - private class MapReduceTaskFuture extends NotifyingFutureImpl { - - private volatile boolean cancelled = false; - - public MapReduceTaskFuture() { - } - - @Override - public boolean cancel(boolean mayInterruptIfRunning) { - if (!isCancelled()) { - RpcManager rpc = cache.getRpcManager(); - synchronized (cancellableTasks) { - for (CancellableTaskPart task : cancellableTasks) { - boolean sendingToSelf = task.getExecutionTarget().equals( - rpc.getTransport().getAddress()); - CancelCommand cc = buildCancelCommand(task); - if (sendingToSelf) { - cc.init(cancellationService); - try { - cc.perform(null); - } catch (Throwable e) { - log.couldNotExecuteCancellationLocally(e.getLocalizedMessage()); - } - } else { - rpc.invokeRemotely(Collections.singletonList(task.getExecutionTarget()), cc, rpcOptionsBuilder.build()); - } - cancelled = true; - } - } - return cancelled; - } else { - //already cancelled - return false; - } - } - - @Override - public boolean isCancelled() { - return cancelled; - } - } - - private abstract class TaskPart implements Future, CancellableTaskPart { - - private Future> f; - private final Address executionTarget; - - public TaskPart(Address executionTarget) { - this.executionTarget = executionTarget; - } - - @Override - public Address getExecutionTarget() { - return executionTarget; - } - - @Override - public boolean cancel(boolean mayInterruptIfRunning) { - return false; - } - - @Override - public boolean isCancelled() { - return false; - } - - @Override - public boolean isDone() { - return false; - } - - @Override - public V get() throws InterruptedException, ExecutionException { - return retrieveResult(f.get()); - } - - protected Address getAddress() { - return clusteringDependentLogic.getAddress(); - } - - protected boolean locallyExecuted(){ - return getAddress().equals(getExecutionTarget()); - } - - public abstract void execute(); - - @SuppressWarnings("unchecked") - private V retrieveResult(Object response) throws ExecutionException { - if (response == null) { - throw new ExecutionException("Execution returned null value", - new NullPointerException()); - } - if (response instanceof Exception) { - throw new ExecutionException((Exception) response); - } - - Map mapResult = (Map) response; - assert mapResult.size() == 1; - for (Entry e : mapResult.entrySet()) { - if (e.getValue() instanceof SuccessfulResponse) { - return (V) ((SuccessfulResponse) e.getValue()).getResponseValue(); - } - } - throw new ExecutionException(new IllegalStateException("Invalid response " + response)); - } - - @Override - public V get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, - TimeoutException { - return retrieveResult(f.get(timeout, unit)); - } - - public void setFuture(Future> future) { - this.f = future; - } - } - - private class MapTaskPart extends TaskPart { - - private final MapCombineCommand mcc; - private final boolean distributedReduce; - - public MapTaskPart(Address executionTarget, MapCombineCommand command, - boolean distributedReduce) { - super(executionTarget); - this.mcc = command; - this.distributedReduce = distributedReduce; - } - - @Override - @SuppressWarnings("unchecked") - public void execute() { - if (locallyExecuted()) { - Callable> callable; - if (distributedReduce) { - callable = () -> { - Set result = invokeMapCombineLocally(); - return Collections.singletonMap(getAddress(), SuccessfulResponse.create(result)); - }; - } else { - callable = () -> { - Map> result = invokeMapCombineLocallyForLocalReduction(); - return Collections.singletonMap(getAddress(), SuccessfulResponse.create(result)); - }; - } - Future> localFuture = mapReduceManager.getExecutorService().submit( - callable); - setFuture(localFuture); - } else { - RpcManager rpc = SecurityActions.getCacheRpcManager(cache); - try { - log.debugf("Invoking %s on %s", mcc, getExecutionTarget()); - CompletableFuture> remoteFuture = rpc.invokeRemotelyAsync( - Collections.singleton(getExecutionTarget()), mcc, rpcOptionsBuilder.build()); - setFuture(remoteFuture); - log.debugf("Invoked %s on %s ", mcc, getExecutionTarget()); - } catch (Exception ex) { - throw new MapReduceException( - "Could not invoke map phase of MapReduceTask on remote node " - + getExecutionTarget(), ex); - } - } - } - - private Map> invokeMapCombineLocallyForLocalReduction() throws InterruptedException { - log.debugf("Invoking %s locally", mcc); - try { - cancellationService.register(Thread.currentThread(), mcc.getUUID()); - mcc.init(mapReduceManager); - return mapReduceManager.mapAndCombineForLocalReduction(mcc); - } finally { - cancellationService.unregister(mcc.getUUID()); - log.debugf("Invoked %s locally", mcc); - } - } - - private Set invokeMapCombineLocally() throws InterruptedException { - log.debugf("Invoking %s locally", mcc); - try { - cancellationService.register(Thread.currentThread(), mcc.getUUID()); - mcc.init(mapReduceManager); - return mapReduceManager.mapAndCombineForDistributedReduction(mcc); - } finally { - cancellationService.unregister(mcc.getUUID()); - log.debugf("Invoked %s locally", mcc); - } - } - - @Override - public UUID getUUID() { - return mcc.getUUID(); - } - } - - private class ReduceTaskPart extends TaskPart { - - private final ReduceCommand rc; - private final String cacheName; - - public ReduceTaskPart(Address executionTarget, ReduceCommand command, - String destinationCacheName) { - super(executionTarget); - this.rc = command; - this.cacheName = destinationCacheName; - } - - @Override - @SuppressWarnings("unchecked") - public void execute() { - if (locallyExecuted()) { - Callable> callable = () -> { - Cache dstCache = cache.getCacheManager().getCache(cacheName); - Map result = invokeReduceLocally(dstCache); - return Collections.singletonMap(getAddress(), SuccessfulResponse.create(result)); - }; - Future> future = mapReduceManager.getExecutorService().submit(callable); - setFuture(future); - } else { - RpcManager rpc = cache.getRpcManager(); - try { - log.debugf("Invoking %s on %s", rc, getExecutionTarget()); - CompletableFuture> future = rpc.invokeRemotelyAsync( - Collections.singleton(getExecutionTarget()), rc, rpcOptionsBuilder.build()); - setFuture(future); - log.debugf("Invoked %s on %s ", rc, getExecutionTarget()); - } catch (Exception ex) { - throw new MapReduceException( - "Could not invoke map phase of MapReduceTask on remote node " - + getExecutionTarget(), ex); - } - } - } - - private Map invokeReduceLocally(Cache dstCache) { - rc.init(mapReduceManager); - Map localReduceResult = null; - try { - log.debugf("Invoking %s locally ", rc); - if(rc.emitsIntoResultingCache()){ - mapReduceManager.reduce(rc, rc.getResultCacheName()); - localReduceResult = Collections.emptyMap(); - } else { - localReduceResult = mapReduceManager.reduce(rc); - } - log.debugf("Invoked %s locally", rc); - } catch (Throwable e1) { - throw new MapReduceException("Could not invoke MapReduce task locally ", e1); - } - return localReduceResult; - } - - @Override - public UUID getUUID() { - return rc.getUUID(); - } - } - - private interface CancellableTaskPart { - UUID getUUID(); - Address getExecutionTarget(); - } -} diff --git a/core/src/main/java/org/infinispan/distexec/mapreduce/Mapper.java b/core/src/main/java/org/infinispan/distexec/mapreduce/Mapper.java deleted file mode 100644 index 86a293e5abb0..000000000000 --- a/core/src/main/java/org/infinispan/distexec/mapreduce/Mapper.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.Cache; -import org.infinispan.CacheSet; - -import java.io.Serializable; - -/** - * Implementation of a Mapper class is a component of a MapReduceTask invoked once for each input - * entry K,V. Every Mapper instance migrated to an Infinispan node, given a cache entry K,V input - * pair transforms that input pair into intermediate keys and emits them into Collector provided by - * Infinispan execution environment. Intermediate results are further reduced using a - * {@link Reducer}. - * - * - * @see Reducer - * @see MapReduceTask - * - * @author Manik Surtani - * @author Vladimir Blagojevic - * @author Sanne Grinovero - * - * @since 5.0 - * @deprecated Map reduce is being replaced by Streams - * @see Cache#entrySet() - * @see CacheSet#stream() - */ -public interface Mapper extends Serializable { - - /** - * Invoked once for each input cache entry KIn,VOut pair. - */ - void map(KIn key, VIn value, Collector collector); -} diff --git a/core/src/main/java/org/infinispan/distexec/mapreduce/Reducer.java b/core/src/main/java/org/infinispan/distexec/mapreduce/Reducer.java deleted file mode 100644 index e930426cc147..000000000000 --- a/core/src/main/java/org/infinispan/distexec/mapreduce/Reducer.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.Cache; -import org.infinispan.CacheSet; - -import java.io.Serializable; -import java.util.Iterator; - -/** - * Reduces intermediate key/value results from map phase of MapReduceTask. Infinispan distributed - * execution environment uses one instance of Reducer per execution node. - * - * - * @see Mapper - * @see MapReduceTask - * - * @author Manik Surtani - * @author Vladimir Blagojevic - * @author Sanne Grinovero - * - * @since 5.0 - * @deprecated Map reduce is being replaced by Streams - * @see Cache#entrySet() - * @see CacheSet#stream() - */ -public interface Reducer extends Serializable { - - /** - * Combines/reduces all intermediate values for a particular intermediate key to a single value. - *

- * - */ - VOut reduce(KOut reducedKey, Iterator iter); - -} diff --git a/core/src/main/java/org/infinispan/distexec/mapreduce/SecurityActions.java b/core/src/main/java/org/infinispan/distexec/mapreduce/SecurityActions.java deleted file mode 100644 index 01d68edbbb18..000000000000 --- a/core/src/main/java/org/infinispan/distexec/mapreduce/SecurityActions.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import java.security.AccessController; -import java.security.PrivilegedAction; -import java.util.List; - -import org.infinispan.AdvancedCache; -import org.infinispan.configuration.cache.Configuration; -import org.infinispan.distribution.DistributionManager; -import org.infinispan.factories.ComponentRegistry; -import org.infinispan.interceptors.base.CommandInterceptor; -import org.infinispan.remoting.rpc.RpcManager; -import org.infinispan.security.AuthorizationManager; -import org.infinispan.security.Security; -import org.infinispan.security.actions.GetCacheAuthorizationManagerAction; -import org.infinispan.security.actions.GetCacheComponentRegistryAction; -import org.infinispan.security.actions.GetCacheConfigurationAction; -import org.infinispan.security.actions.GetCacheDistributionManagerAction; -import org.infinispan.security.actions.GetCacheInterceptorChainAction; -import org.infinispan.security.actions.GetCacheRpcManagerAction; - -/** - * SecurityActions for the org.infinispan.distexec.mapreduce package. - * - * Do not move. Do not change class and method visibility to avoid being called from other - * {@link java.security.CodeSource}s, thus granting privilege escalation to external code. - * - * @author Tristan Tarrant - * @since 7.0 - */ -final class SecurityActions { - private static T doPrivileged(PrivilegedAction action) { - if (System.getSecurityManager() != null) { - return AccessController.doPrivileged(action); - } else { - return Security.doPrivileged(action); - } - } - - static ComponentRegistry getCacheComponentRegistry(final AdvancedCache cache) { - GetCacheComponentRegistryAction action = new GetCacheComponentRegistryAction(cache); - return doPrivileged(action); - } - - static AuthorizationManager getCacheAuthorizationManager(final AdvancedCache cache) { - GetCacheAuthorizationManagerAction action = new GetCacheAuthorizationManagerAction(cache); - return doPrivileged(action); - } - - static RpcManager getCacheRpcManager(final AdvancedCache cache) { - GetCacheRpcManagerAction action = new GetCacheRpcManagerAction(cache); - return doPrivileged(action); - } - - static DistributionManager getCacheDistributionManager(final AdvancedCache cache) { - GetCacheDistributionManagerAction action = new GetCacheDistributionManagerAction(cache); - return doPrivileged(action); - } - - static Configuration getCacheConfiguration(final AdvancedCache cache) { - GetCacheConfigurationAction action = new GetCacheConfigurationAction(cache); - return doPrivileged(action); - } - - static List getInterceptorChain(final AdvancedCache cache) { - GetCacheInterceptorChainAction action = new GetCacheInterceptorChainAction(cache); - return doPrivileged(action); - } -} diff --git a/core/src/main/java/org/infinispan/distexec/mapreduce/package-info.java b/core/src/main/java/org/infinispan/distexec/mapreduce/package-info.java deleted file mode 100644 index 789102124e93..000000000000 --- a/core/src/main/java/org/infinispan/distexec/mapreduce/package-info.java +++ /dev/null @@ -1,6 +0,0 @@ -/** - * Map/Reduce APIs - * - * @public - */ -package org.infinispan.distexec.mapreduce; \ No newline at end of file diff --git a/core/src/main/java/org/infinispan/distexec/mapreduce/spi/DefaultMapReduceTaskLifecycle.java b/core/src/main/java/org/infinispan/distexec/mapreduce/spi/DefaultMapReduceTaskLifecycle.java deleted file mode 100644 index b3f0e10c6ffc..000000000000 --- a/core/src/main/java/org/infinispan/distexec/mapreduce/spi/DefaultMapReduceTaskLifecycle.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.infinispan.distexec.mapreduce.spi; - -import org.infinispan.Cache; -import org.infinispan.distexec.mapreduce.Mapper; -import org.infinispan.distexec.mapreduce.Reducer; -import org.kohsuke.MetaInfServices; - -@MetaInfServices -public class DefaultMapReduceTaskLifecycle implements MapReduceTaskLifecycle { - - @Override - public void onPreExecute(Mapper mapper, Cache inputCache) { - // intentionally no-op - } - - @Override - public void onPostExecute(Mapper mapper) { - // intentionally no-op - } - - @Override - public void onPreExecute(Reducer reducer, Cache inputCache) { - // intentionally no-op - } - - @Override - public void onPostExecute(Reducer reducer) { - // intentionally no-op - } - -} diff --git a/core/src/main/java/org/infinispan/distexec/mapreduce/spi/MapReduceTaskLifecycle.java b/core/src/main/java/org/infinispan/distexec/mapreduce/spi/MapReduceTaskLifecycle.java deleted file mode 100644 index 7cfa15fb8ec3..000000000000 --- a/core/src/main/java/org/infinispan/distexec/mapreduce/spi/MapReduceTaskLifecycle.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.infinispan.distexec.mapreduce.spi; - -import org.infinispan.Cache; -import org.infinispan.distexec.mapreduce.Mapper; -import org.infinispan.distexec.mapreduce.Reducer; - -public interface MapReduceTaskLifecycle { - - void onPreExecute(Mapper mapper, Cache inputCache); - - void onPostExecute(Mapper mapper); - - void onPreExecute(Reducer reducer, Cache inputCache); - - void onPostExecute(Reducer reducer); -} diff --git a/core/src/main/java/org/infinispan/distexec/mapreduce/spi/MapReduceTaskLifecycleService.java b/core/src/main/java/org/infinispan/distexec/mapreduce/spi/MapReduceTaskLifecycleService.java deleted file mode 100644 index 5f0f83b53a74..000000000000 --- a/core/src/main/java/org/infinispan/distexec/mapreduce/spi/MapReduceTaskLifecycleService.java +++ /dev/null @@ -1,82 +0,0 @@ -package org.infinispan.distexec.mapreduce.spi; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.ServiceConfigurationError; - -import org.infinispan.Cache; -import org.infinispan.commons.util.ServiceFinder; -import org.infinispan.distexec.mapreduce.Mapper; -import org.infinispan.distexec.mapreduce.Reducer; -import org.infinispan.util.logging.Log; -import org.infinispan.util.logging.LogFactory; - -public final class MapReduceTaskLifecycleService { - - private static final Log log = LogFactory.getLog(MapReduceTaskLifecycleService.class); - private static MapReduceTaskLifecycleService service; - private List lifecycles; - - private MapReduceTaskLifecycleService() { - lifecycles = new ArrayList(); - for (MapReduceTaskLifecycle l : ServiceFinder.load(MapReduceTaskLifecycle.class)) { - lifecycles.add(l); - } - } - - public static synchronized MapReduceTaskLifecycleService getInstance() { - if (service == null) { - service = new MapReduceTaskLifecycleService(); - } - return service; - } - - public void onPreExecute(Mapper mapper, Cache inputCache) { - try { - for (MapReduceTaskLifecycle l : lifecycles) { - l.onPreExecute(mapper, inputCache); - } - } catch (ServiceConfigurationError serviceError) { - log.errorReadingProperties(new IOException( - "Could not properly load and instantiate DistributedTaskLifecycle service ", - serviceError)); - } - } - - public void onPostExecute(Mapper mapper) { - try { - for (MapReduceTaskLifecycle l : lifecycles) { - l.onPostExecute(mapper); - } - } catch (ServiceConfigurationError serviceError) { - log.errorReadingProperties(new IOException( - "Could not properly load and instantiate DistributedTaskLifecycle service ", - serviceError)); - } - } - - public void onPreExecute(Reducer reducer, Cache inputCache) { - try { - for (MapReduceTaskLifecycle l : lifecycles) { - l.onPreExecute(reducer, inputCache); - } - } catch (ServiceConfigurationError serviceError) { - log.errorReadingProperties(new IOException( - "Could not properly load and instantiate DistributedTaskLifecycle service ", - serviceError)); - } - } - - public void onPostExecute(Reducer reducer) { - try { - for (MapReduceTaskLifecycle l : lifecycles) { - l.onPostExecute(reducer); - } - } catch (ServiceConfigurationError serviceError) { - log.errorReadingProperties(new IOException( - "Could not properly load and instantiate DistributedTaskLifecycle service ", - serviceError)); - } - } -} \ No newline at end of file diff --git a/core/src/main/java/org/infinispan/interceptors/compat/BaseTypeConverterInterceptor.java b/core/src/main/java/org/infinispan/interceptors/compat/BaseTypeConverterInterceptor.java index 781305ab757b..1c6877e22762 100644 --- a/core/src/main/java/org/infinispan/interceptors/compat/BaseTypeConverterInterceptor.java +++ b/core/src/main/java/org/infinispan/interceptors/compat/BaseTypeConverterInterceptor.java @@ -286,7 +286,7 @@ protected CacheStream getStream(boolean parallel) { () -> StreamSupport.stream(closeableSpliterator, parallel)), parallel, cache.getAdvancedCache().getComponentRegistry()); // We rely on the fact that on close returns the same instance - stream.onClose(() -> closeableSpliterator.close()); + stream.onClose(closeableSpliterator::close); return new TypeConverterStream(stream, converter, entryFactory); } }; @@ -324,7 +324,7 @@ protected CacheStream> getStream(boolean parallel) { () -> StreamSupport.stream(closeableSpliterator, parallel)), parallel, cache.getAdvancedCache().getComponentRegistry()); // We rely on the fact that on close returns the same instance - stream.onClose(() -> closeableSpliterator.close()); + stream.onClose(closeableSpliterator::close); return new TypeConverterStream(stream, converter, entryFactory); } }; diff --git a/core/src/main/java/org/infinispan/marshall/core/ExternalizerTable.java b/core/src/main/java/org/infinispan/marshall/core/ExternalizerTable.java index 1bd89b81e380..db1ef8696421 100644 --- a/core/src/main/java/org/infinispan/marshall/core/ExternalizerTable.java +++ b/core/src/main/java/org/infinispan/marshall/core/ExternalizerTable.java @@ -41,7 +41,6 @@ import org.infinispan.container.versioning.NumericVersion; import org.infinispan.container.versioning.SimpleClusteredVersion; import org.infinispan.context.Flag; -import org.infinispan.distexec.mapreduce.MapReduceManagerImpl; import org.infinispan.distribution.ch.impl.AffinityPartitioner; import org.infinispan.distribution.ch.impl.DefaultConsistentHash; import org.infinispan.distribution.ch.impl.DefaultConsistentHashFactory; @@ -352,8 +351,6 @@ private void loadInternalMarshallables() { addInternalExternalizer(new XSiteState.XSiteStateExternalizer()); addInternalExternalizer(new CompositeKeyFilter.Externalizer()); addInternalExternalizer(new CompositeKeyValueFilter.Externalizer()); - addInternalExternalizer(new MapReduceManagerImpl.DeltaListExternalizer()); - addInternalExternalizer(new MapReduceManagerImpl.DeltaAwareListExternalizer()); addInternalExternalizer(new CacheStatusResponse.Externalizer()); addInternalExternalizer(new CacheEventConverterAsConverter.Externalizer()); addInternalExternalizer(new CacheEventFilterAsKeyValueFilter.Externalizer()); diff --git a/core/src/main/java/org/infinispan/marshall/exts/CacheRpcCommandExternalizer.java b/core/src/main/java/org/infinispan/marshall/exts/CacheRpcCommandExternalizer.java index 6d3c0d46989a..4abb664e2a36 100644 --- a/core/src/main/java/org/infinispan/marshall/exts/CacheRpcCommandExternalizer.java +++ b/core/src/main/java/org/infinispan/marshall/exts/CacheRpcCommandExternalizer.java @@ -5,8 +5,6 @@ import org.infinispan.commands.RemoveCacheCommand; import org.infinispan.commands.control.LockControlCommand; import org.infinispan.commands.read.DistributedExecuteCommand; -import org.infinispan.commands.read.MapCombineCommand; -import org.infinispan.commands.read.ReduceCommand; import org.infinispan.commands.remote.CacheRpcCommand; import org.infinispan.commands.remote.ClusteredGetAllCommand; import org.infinispan.commands.remote.ClusteredGetCommand; @@ -89,8 +87,8 @@ public CacheRpcCommandExternalizer(GlobalComponentRegistry gcr, ReplicableComman @Override public Set> getTypeClasses() { //noinspection unchecked - Set> coreCommands = Util.asSet(MapCombineCommand.class, - ReduceCommand.class, DistributedExecuteCommand.class, LockControlCommand.class, + Set> coreCommands = Util.asSet(DistributedExecuteCommand.class, + LockControlCommand.class, StateRequestCommand.class, StateResponseCommand.class, ClusteredGetCommand.class, MultipleRpcCommand.class, SingleRpcCommand.class, CommitCommand.class, PrepareCommand.class, RollbackCommand.class, RemoveCacheCommand.class, diff --git a/core/src/main/java/org/infinispan/util/logging/Log.java b/core/src/main/java/org/infinispan/util/logging/Log.java index 504cac6ff24b..311d5b83ae3f 100644 --- a/core/src/main/java/org/infinispan/util/logging/Log.java +++ b/core/src/main/java/org/infinispan/util/logging/Log.java @@ -1322,9 +1322,6 @@ void preparedTxAlreadyExists(RecoveryAwareTransaction previous, @Message(value = "Invocation batching not enabled in current configuration! Please enable it.", id = 378) CacheConfigurationException invocationBatchingNotEnabled(); - @Message(value = "Map Reduce Framework is not supported in simple cache", id = 379) - CacheConfigurationException mapReduceNotSupported(); - @Message(value = "Distributed Executors Framework is not supported in simple cache", id = 380) CacheConfigurationException distributedExecutorsNotSupported(); diff --git a/core/src/test/java/org/infinispan/api/SimpleCacheTest.java b/core/src/test/java/org/infinispan/api/SimpleCacheTest.java index 4db19b136a4c..37e26f68090a 100644 --- a/core/src/test/java/org/infinispan/api/SimpleCacheTest.java +++ b/core/src/test/java/org/infinispan/api/SimpleCacheTest.java @@ -8,7 +8,6 @@ import org.infinispan.configuration.cache.ConfigurationBuilder; import org.infinispan.configuration.cache.Index; import org.infinispan.distexec.DefaultExecutorService; -import org.infinispan.distexec.mapreduce.MapReduceTask; import org.infinispan.interceptors.base.BaseCustomInterceptor; import org.infinispan.manager.EmbeddedCacheManager; import org.infinispan.test.fwk.TestCacheManagerFactory; @@ -38,11 +37,6 @@ public void testAddInterceptor() { cache().getAdvancedCache().addInterceptor(new CustomInterceptorConfigTest.DummyInterceptor(), 0); } - @Test(expectedExceptions = CacheConfigurationException.class) - public void testMapReduce() { - new MapReduceTask<>(cache()).execute(); - } - @Test(expectedExceptions = CacheConfigurationException.class) public void testDistributedExecutor() { new DefaultExecutorService(cache()).submit(() -> null); diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/BaseLargeWordCountMapReduceTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/BaseLargeWordCountMapReduceTest.java deleted file mode 100644 index 0fc3a4f5e8b8..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/BaseLargeWordCountMapReduceTest.java +++ /dev/null @@ -1,219 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import static org.junit.Assert.assertTrue; - -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; -import java.util.StringTokenizer; - -import org.infinispan.Cache; -import org.infinispan.commons.util.FileLookup; -import org.infinispan.commons.util.FileLookupFactory; -import org.infinispan.configuration.cache.CacheMode; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.infinispan.test.MultipleCacheManagersTest; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -/** - * BaseLargeWordCountMapReduceTest tests parallel execution of Map/Reduce using word count over - * Shakespeare's Macbeth :-) - * - * @author Vladimir Blagojevic - * @since 7.0 - */ -@Test(groups = "stress", testName = "distexec.mapreduce.BaseLargeWordCountMapReduceTest") -public abstract class BaseLargeWordCountMapReduceTest extends MultipleCacheManagersTest { - - protected HashMap counts = new HashMap(); - - public BaseLargeWordCountMapReduceTest() { - cleanup = CleanupPhase.AFTER_TEST; - } - - protected CacheMode getCacheMode() { - return CacheMode.DIST_SYNC; - } - - protected String cacheName() { - return "largemapreducecache"; - } - - @BeforeClass(alwaysRun = true) - public void createBeforeClass() throws Throwable { - super.createBeforeClass(); - specifyWordCounts(); - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - protected MapReduceTask createMapReduceTask(Cache c) { - return new MapReduceTask(c); - } - - public void testInvokeMapReduceOnAllKeys() throws Exception { - MapReduceTask task = invokeMapReduce(null); - Map mapReduce = task.execute(); - verifyResults(mapReduce); - } - - public void testInvokeMapReduceOnAllKeysWithResultCache() throws Exception { - String cacheName = "resultCache"; - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(CacheMode.REPL_SYNC, true); - defineConfigurationOnAllManagers(cacheName, builder); - try { - MapReduceTask task = invokeMapReduce(null); - Cache c1 = cache(0, cacheName); - task.execute(c1); - verifyResults(c1); - c1.clear(); - } finally { - removeCacheFromCluster(cacheName); - } - } - - public void testInvokeMapReduceOnSubsetOfKeys() throws Exception { - MapReduceTask task = invokeMapReduce(new String[] { "1", "2", "3" }); - Map mapReduce = task.execute(); - assertPartialWordCount(countWords(mapReduce)); - } - - public void testInvokeMapReduceOnSubsetOfKeysWithResultCache() throws Exception { - String cacheName = "resultCache2"; - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(CacheMode.REPL_SYNC, true); - defineConfigurationOnAllManagers(cacheName, builder); - try { - MapReduceTask task = invokeMapReduce(new String[] { "1", "2", "3" }); - task.execute(cacheName); - Cache c1 = cache(0, cacheName); - assertPartialWordCount(countWords(c1)); - c1.clear(); - } finally { - removeCacheFromCluster(cacheName); - } - } - - protected MapReduceTask invokeMapReduce(String keys[], - Mapper mapper, Reducer reducer) throws Exception { - return invokeMapReduce(keys, mapper, reducer, true); - } - - protected MapReduceTask invokeMapReduce(String keys[]) throws Exception { - return invokeMapReduce(keys, true); - } - - protected MapReduceTask invokeMapReduce(String keys[], boolean useCombiner) - throws Exception { - return invokeMapReduce(keys, new WordCountMapper(), new WordCountReducer(), useCombiner); - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - protected MapReduceTask invokeMapReduce(String keys[], - Mapper mapper, Reducer reducer, boolean useCombiner) - throws Exception { - - Cache c1 = cache(0, cacheName()); - Cache c2 = cache(1, cacheName()); - FileLookup fileLookup = FileLookupFactory.newInstance(); - InputStream is = fileLookup.lookupFile("mapreduce/macbeth.txt", getClass().getClassLoader()); - - BufferedReader br = new BufferedReader(new InputStreamReader(is)); - String line = null; - int lineCount = 0; - while ((line = br.readLine()) != null) { - if (Math.random() > 0.5) { - c1.put(String.valueOf(lineCount), line); - } else { - c2.put(String.valueOf(lineCount), line); - } - lineCount++; - } - MapReduceTask task = createMapReduceTask(c1); - task.mappedWith(mapper).reducedWith(reducer); - if (useCombiner) - task.combinedWith(reducer); - - if (keys != null && keys.length > 0) { - task.onKeys(keys); - } - return task; - } - - protected void specifyWordCounts() { - counts.put("brave", 2); - counts.put("more.", 2); - counts.put("over-credulous", 1); - counts.put("the", 607); - counts.put("MACBETH", 241); - } - - protected void verifyResults(Map result) { - verifyResults(result, counts); - } - - protected void verifyResults(Map result, Map verifyAgainst) { - assertTrue("Results should have at least 1 answer", result.size() > 0); - for (Entry e : result.entrySet()) { - String key = e.getKey(); - Integer value = e.getValue(); - Integer count = verifyAgainst.get(key); - if (count == null) - continue; - assertTrue("key '" + key + "' does not have count " + count + " but " + value, count.equals(value)); - } - } - - protected int countWords(Map result) { - int sum = 0; - for (Entry e : result.entrySet()) { - sum += e.getValue(); - } - return sum; - } - - protected void assertTotalWordCount(int actualWordCount) { - int expectedWordCount = 18299; - assertTrue(" word count of " + actualWordCount + " incorrect , expected " + expectedWordCount, - actualWordCount == expectedWordCount); - } - - protected void assertPartialWordCount(int actualWordCount) { - int expectedWordCount = 2; - assertTrue(" word count of " + actualWordCount + " incorrect , expected " + expectedWordCount, - actualWordCount == expectedWordCount); - } - - static class WordCountMapper implements Mapper { - /** The serialVersionUID */ - private static final long serialVersionUID = -5943370243108735560L; - - @Override - public void map(String key, String value, Collector collector) { - if (value == null) - throw new IllegalArgumentException("Key " + key + " has value " + value); - StringTokenizer tokens = new StringTokenizer(value); - while (tokens.hasMoreElements()) { - String s = (String) tokens.nextElement(); - collector.emit(s, 1); - } - } - } - - static class WordCountReducer implements Reducer { - /** The serialVersionUID */ - private static final long serialVersionUID = 1901016598354633256L; - - @Override - public Integer reduce(String key, Iterator iter) { - int sum = 0; - while (iter.hasNext()) { - sum += iter.next(); - } - return sum; - } - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/BaseWordCountMapReduceTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/BaseWordCountMapReduceTest.java deleted file mode 100644 index 8b68ac0cfa1a..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/BaseWordCountMapReduceTest.java +++ /dev/null @@ -1,437 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.Cache; -import org.infinispan.commons.CacheException; -import org.infinispan.configuration.cache.CacheMode; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.infinispan.test.MultipleCacheManagersTest; -import org.junit.Assert; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; -import java.util.StringTokenizer; -import java.util.concurrent.Future; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * BaseTest for MapReduceTask - * - * @author Vladimir Blagojevic - */ -@Test(groups = "functional", testName = "distexec.mapreduce.BaseWordCountMapReduceTest") -public abstract class BaseWordCountMapReduceTest extends MultipleCacheManagersTest { - - protected static HashMap counts = new HashMap(); - - public BaseWordCountMapReduceTest() { - cleanup = CleanupPhase.AFTER_TEST; - } - - protected CacheMode getCacheMode() { - return CacheMode.DIST_SYNC; - } - - protected String cacheName(){ - return "mapreducecache"; - } - - @Override - @BeforeClass(alwaysRun = true) - public void createBeforeClass() throws Throwable { - super.createBeforeClass(); - specifyWordCounts(); - } - - protected static void specifyWordCounts() { - counts.put("of", 2); - counts.put("open", 1); - counts.put("is", 6); - counts.put("source", 1); - counts.put("JBoss", 5); - counts.put("in", 2); - counts.put("capital", 1); - counts.put("world", 3); - counts.put("Hello", 2); - counts.put("Ontario", 1); - counts.put("cool", 1); - counts.put("JUDCon", 2); - counts.put("Infinispan", 3); - counts.put("a", 1); - counts.put("awesome", 1); - counts.put("Application", 1); - counts.put("am", 1); - counts.put("RedHat", 2); - counts.put("Server", 1); - counts.put("community", 2); - counts.put("as", 1); - counts.put("the", 1); - counts.put("Toronto", 2); - counts.put("close", 1); - counts.put("to", 1); - counts.put("division", 1); - counts.put("here", 1); - counts.put("Boston", 3); - counts.put("well", 1); - counts.put("World", 2); - counts.put("I", 1); - counts.put("rules", 2); - } - - /** - * Test helper - has to be public because it used in CDI module - * - */ - @SuppressWarnings({ "rawtypes", "unchecked" }) - protected MapReduceTask createMapReduceTask(Cache c){ - return new MapReduceTask(c); - } - - - /** - * Test helper - has to be public because it used in CDI module - * - */ - public MapReduceTask invokeMapReduce(String keys[], - Mapper mapper, Reducer reducer) - throws Exception { - return invokeMapReduce(keys, mapper, reducer, true); - } - - /** - * Test helper - has to be public because it used in CDI module - * - */ - @SuppressWarnings({ "rawtypes", "unchecked" }) - public MapReduceTask invokeMapReduce(String keys[], - Mapper mapper, Reducer reducer, boolean useCombiner) - throws Exception { - Cache c1 = cache(0, cacheName()); - Cache c2 = cache(1, cacheName()); - - c1.put("1", "Hello world here I am"); - c2.put("2", "Infinispan rules the world"); - c1.put("3", "JUDCon is in Boston"); - c2.put("4", "JBoss World is in Boston as well"); - c1.put("12","JBoss Application Server"); - c2.put("15", "Hello world"); - c1.put("14", "Infinispan community"); - - c1.put("111", "Infinispan open source"); - c2.put("112", "Boston is close to Toronto"); - c1.put("113", "Toronto is a capital of Ontario"); - c2.put("114", "JUDCon is cool"); - c1.put("211", "JBoss World is awesome"); - c2.put("212", "JBoss rules"); - c1.put("213", "JBoss division of RedHat "); - c2.put("214", "RedHat community"); - - MapReduceTask task = createMapReduceTask(c1); - task.mappedWith(mapper).reducedWith(reducer); - if(useCombiner) - task.combinedWith(reducer); - - if(keys != null && keys.length>0){ - task.onKeys(keys); - } - return task; - } - - /** - * Test helper - has to be public because it used in CDI module - * - */ - public MapReduceTask invokeMapReduce(String keys[]) throws Exception{ - return invokeMapReduce(keys, true); - } - - /** - * Test helper - has to be public because it used in CDI module - * - */ - public MapReduceTask invokeMapReduce(String keys[], boolean useCombiner) throws Exception{ - return invokeMapReduce(keys,new WordCountMapper(), new WordCountReducer(), useCombiner); - } - - public void testInvokeMapReduceOnAllKeys() throws Exception { - MapReduceTask task = invokeMapReduce(null); - Map mapReduce = task.execute(); - verifyResults(mapReduce); - } - - public void testInvokeMapReduceOnAllKeysWithResultCache() throws Exception { - String cacheName = "resultCache"; - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(CacheMode.REPL_SYNC, true); - defineConfigurationOnAllManagers(cacheName, builder); - try { - MapReduceTask task = invokeMapReduce(null); - Cache c1 = cache(0, cacheName); - task.execute(c1); - verifyResults(c1); - c1.clear(); - } finally { - removeCacheFromCluster(cacheName); - } - } - - public void testInvokeMapReduceOnEmptyKeys() throws Exception { - MapReduceTask task = invokeMapReduce(new String[] {}); - Map mapReduce = task.execute(); - verifyResults(mapReduce); - } - - public void testInvokeMapReduceOnAllKeysWithCombiner() throws Exception { - MapReduceTask task = invokeMapReduce(null, true); - Map mapReduce = task.execute(); - verifyResults(mapReduce); - } - - public void testCombinerDoesNotChangeResult() throws Exception { - MapReduceTask task = invokeMapReduce(null, true); - Map mapReduce = task.execute(); - - - MapReduceTask task2 = invokeMapReduce(null, false); - Map mapReduce2 = task2.execute(); - - assertEquals(mapReduce2.get("Infinispan"), mapReduce.get("Infinispan")); - assertEquals(mapReduce2.get("RedHat"), mapReduce.get("RedHat")); - } - - /** - * Tests isolation as mapper and reducer get invoked across the cluster - * https://issues.jboss.org/browse/ISPN-1041 - * - * @throws Exception - */ - public void testMapperReducerIsolation() throws Exception{ - invokeMapReduce(null, new IsolationMapper(), new IsolationReducer(), false); - } - - public void testInvokeMapReduceOnAllKeysAsync() throws Exception { - MapReduceTask task = invokeMapReduce(null); - Future> future = task.executeAsynchronously(); - Map mapReduce = future.get(); - verifyResults(mapReduce); - } - - public void testInvokeMapReduceOnSubsetOfKeys() throws Exception { - MapReduceTask task = invokeMapReduce(new String[]{"1", "2", "3"}); - Map mapReduce = task.execute(); - assertPartialWordCount(countWords(mapReduce)); - } - - public void testInvokeMapReduceOnSubsetOfKeysWithResultCache() throws Exception { - String cacheName = "resultCache2"; - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(CacheMode.REPL_SYNC, true); - defineConfigurationOnAllManagers(cacheName, builder); - try { - MapReduceTask task = invokeMapReduce(new String[] { "1", "2", "3" }); - task.execute(cacheName); - Cache c1 = cache(0, cacheName); - assertPartialWordCount(countWords(c1)); - c1.clear(); - } finally { - removeCacheFromCluster(cacheName); - } - } - - public void testInvokeMapReduceOnSubsetOfKeysAsync() throws Exception { - MapReduceTask task = invokeMapReduce(new String[]{"1", "2", "3"}); - Future> future = task.executeAsynchronously(); - Map mapReduce = future.get(); - assertPartialWordCount(countWords(mapReduce)); - } - - protected void verifyResults(Map result, Map verifyAgainst) { - assertTrue("Results should have at least 1 answer", result.size() > 0); - for (Entry e : result.entrySet()) { - String key = e.getKey(); - Integer count = verifyAgainst.get(key); - assertTrue("key '" + e.getKey() + "' does not have count " + count + " but " + e.getValue(), count.equals(e.getValue())); - } - } - - protected int nodeCount(){ - return getCacheManagers().size(); - } - - public void testInvokeMapReduceOnAllKeysWithCollator() throws Exception { - MapReduceTask task = invokeMapReduce(null); - Integer totalWords = task.execute(new Collator() { - - @Override - public Integer collate(Map reducedResults) { - int sum = 0; - for (Entry e : reducedResults.entrySet()) { - sum += e.getValue(); - } - return sum; - } - }); - assertTotalWordCount(totalWords); - } - - public void testInvokeMapReduceOnSubsetOfKeysWithCollator() throws Exception { - MapReduceTask task = invokeMapReduce(new String[] { "1", "2", "3" }); - Integer totalWords = task.execute(new Collator() { - - @Override - public Integer collate(Map reducedResults) { - int sum = 0; - for (Entry e : reducedResults.entrySet()) { - sum += e.getValue(); - } - return sum; - } - }); - assertPartialWordCount(totalWords); - } - - public void testInvokeMapReduceOnAllKeysWithCollatorAsync() throws Exception { - MapReduceTask task = invokeMapReduce(null); - Future future = task.executeAsynchronously(new Collator() { - - @Override - public Integer collate(final Map reducedResults) { - log.tracef("Executing collator"); - int sum = 0; - for (Entry e : reducedResults.entrySet()) { - sum += e.getValue(); - } - return sum; - } - }); - Integer totalWords = future.get(); - assertTotalWordCount(totalWords); - } - - public void testInvokeMapReduceOnSubsetOfKeysWithCollatorAsync() throws Exception { - MapReduceTask task = invokeMapReduce(new String[] { "1", "2", "3" }); - Future future = task.executeAsynchronously(new Collator() { - - @Override - public Integer collate(Map reducedResults) { - int sum = 0; - for (Entry e : reducedResults.entrySet()) { - sum += e.getValue(); - } - return sum; - } - }); - Integer totalWords = future.get(); - assertPartialWordCount(totalWords); - } - - @Test - public void testCombinerForDistributedReductionWithException() throws Exception { - MapReduceTask task = invokeMapReduce(null); - task.combinedWith(new SimulatedRuntimeExceptionReducer()); - try { - task.execute(); - Assert.fail("Should have thrown exception"); - } catch (Exception e) { - Assert.assertTrue("Exception thrown should be CacheException but it is " + e, e instanceof CacheException); - Throwable cause = e.getCause(); - while (cause.getCause() != null) { - cause = cause.getCause(); - } - Assert.assertTrue("Exception thrown should be RuntimeException but it is " + cause, - cause instanceof RuntimeException); - } - } - - private static class SimulatedRuntimeExceptionReducer implements Reducer { - /** - * - */ - private static final long serialVersionUID = -6208228105751231985L; - - @Override - public Integer reduce(String reducedKey, Iterator iter) { - throw new RuntimeException("Simulated exception"); - } - } - - protected void assertPartialWordCount(int actualWordCount){ - int expectedWordCount= 13; - assertTrue(" word count of " + actualWordCount + " incorrect , expected " + expectedWordCount, actualWordCount == expectedWordCount); - } - - protected void assertTotalWordCount(int actualWordCount){ - int expectedWordCount= 56; - assertTrue(" word count of " + actualWordCount + " incorrect , expected " + expectedWordCount, actualWordCount == expectedWordCount); - } - - protected int countWords(Map result) { - int sum = 0; - for (Entry e : result.entrySet()) { - sum += e.getValue(); - } - return sum; - } - - protected void verifyResults(Map result) { - verifyResults(result, counts); - } - - static class WordCountMapper implements Mapper { - /** The serialVersionUID */ - private static final long serialVersionUID = -5943370243108735560L; - - @Override - public void map(String key, String value, Collector collector) { - if(value == null) throw new IllegalArgumentException("Key " + key + " has value " + value); - StringTokenizer tokens = new StringTokenizer(value); - while (tokens.hasMoreElements()) { - String s = (String) tokens.nextElement(); - collector.emit(s, 1); - } - } - } - - static class WordCountReducer implements Reducer { - /** The serialVersionUID */ - private static final long serialVersionUID = 1901016598354633256L; - - @Override - public Integer reduce(String key, Iterator iter) { - int sum = 0; - while (iter.hasNext()) { - sum += iter.next(); - } - return sum; - } - } - - private static class IsolationMapper implements Mapper { - /** The serialVersionUID */ - private static final long serialVersionUID = 1993535517358319862L; - private int count = 0; - - - @Override - public void map(String key, String value, Collector collector) { - assertEquals(0, count); - count++; - } - } - - private static class IsolationReducer implements Reducer { - /** The serialVersionUID */ - private static final long serialVersionUID = 6069777605143824777L; - private int count = 0; - - @Override - public Integer reduce(String key, Iterator iter) { - assertEquals(0, count); - count++; - return count; - } - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/Book.java b/core/src/test/java/org/infinispan/distexec/mapreduce/Book.java deleted file mode 100644 index 1a7fbe2a6137..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/Book.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import java.io.Serializable; - -/** - * Example object to run a query using MapReduce. - * Using Java here, but it could be anything such - * as free text, JSON, YAML, ... - * - * @author Sanne Grinovero (C) 2011 Red Hat Inc. - */ -public class Book implements Serializable { - - /** The serialVersionUID */ - private static final long serialVersionUID = 201181615056852234L; - - final String title; - final String author; - final String editor; - - public Book(String title, String author, String editor) { - this.title = title; - this.author = author; - this.editor = editor; - } - -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/BookSearchTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/BookSearchTest.java deleted file mode 100644 index 78b7dab27003..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/BookSearchTest.java +++ /dev/null @@ -1,85 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import java.util.Iterator; -import java.util.Map; - -import org.infinispan.Cache; -import org.infinispan.configuration.cache.CacheMode; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.infinispan.test.MultipleCacheManagersTest; -import org.testng.annotations.Test; - -/** - * Example for simple Map Reduce use case. - * The test is marked as abstract for applying different configurations on it. - * - * @author Sanne Grinovero (C) 2011 Red Hat Inc. - */ -@Test(groups = "functional", testName = "distexec.BookSearchTest") -public abstract class BookSearchTest extends MultipleCacheManagersTest { - - @SuppressWarnings({ "rawtypes", "unchecked" }) - public void testBookSearch() { - Cache c1 = cache(0, "bookSearch"); - c1.put("1", - new Book("Seam in Action", - "Dan Allen", - "Manning")); - c1.put("2", - new Book("Hibernate Search in Action", - "Emmanuel Bernard and John Griffin", - "Manning")); - c1.put("3", - new Book("Metaprogramming Ruby", - "Paolo Perrotta", - "The Pragmatic Programmers")); - for (int i = 0; i < 4; i++) { - verifySearch( cache( i, "bookSearch" ) ); - } - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - private void verifySearch(Cache cache) { - - MapReduceTask queryTask = new MapReduceTask(cache); - - queryTask - .mappedWith( new TitleBookSearcher( "Hibernate Search in Action" ) ) - .reducedWith( new BookReducer() ); - - Map queryResult = queryTask.execute(); - assert queryResult.size() == 1; - assert "Hibernate Search in Action".equals( queryResult.values().iterator().next().title ); - } - - static class TitleBookSearcher implements Mapper { - - /** The serialVersionUID */ - private static final long serialVersionUID = -7443288752468217500L; - final String title; - - public TitleBookSearcher(String title) { - this.title = title; - } - - @Override - public void map(String key, Book value, Collector collector) { - if ( title.equals( value.title ) ) { - collector.emit( key, value ); - } - } - } - - static class BookReducer implements Reducer { - - /** The serialVersionUID */ - private static final long serialVersionUID = 5686049814166522660L; - - @Override - public Book reduce(String reducedKey, Iterator iter) { - return iter.next(); - } - - } - -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/BookStoreAsBinarySearchTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/BookStoreAsBinarySearchTest.java deleted file mode 100644 index 98e2dc04e62b..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/BookStoreAsBinarySearchTest.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.configuration.cache.CacheMode; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.testng.annotations.Test; - -/** - * Adding another configuration to BookSearchTest so that the keys and values are stored binary in the cache. - * The test verifies the issue ISPN-2138, i.e. no ClassCasting exception should be thrown. - * - * @author Anna Manukyan - */ -@Test(groups = "functional", testName = "distexec.mapreduce.BookStoreAsBinarySearchTest") -public class BookStoreAsBinarySearchTest extends BookSearchTest { - - @Override - protected void createCacheManagers() { - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(CacheMode.DIST_SYNC, true); - - //Verification for ISPN-2138 - the following exception should not appear: - //java.lang.ClassCastException: org.infinispan.marshall.MarshalledValue cannot be cast to org.infinispan.distexec.mapreduce.Book - builder.storeAsBinary().enable(); - createClusteredCaches(4, "bookSearch", builder); - } - -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/BookStoreAsBinaryWithStoreSearchTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/BookStoreAsBinaryWithStoreSearchTest.java deleted file mode 100644 index c798d922a835..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/BookStoreAsBinaryWithStoreSearchTest.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.configuration.cache.CacheMode; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.infinispan.eviction.EvictionStrategy; -import org.infinispan.persistence.dummy.DummyInMemoryStoreConfigurationBuilder; -import org.testng.annotations.Test; - -/** - * Adding another configuration to BookSearchTest so that the keys and values are stored binary in the cache. - * The cache is configured with eviction and passivation is done to cache store. - * - * The test verifies the issue ISPN-2386, i.e. no ClassCastException should be thrown. - * - * @author Anna Manukyan - */ -@Test(groups = "functional", testName = "distexec.mapreduce.BookStoreAsBinaryWithStoreSearchTest") -public class BookStoreAsBinaryWithStoreSearchTest extends BookSearchTest { - - @Override - protected void createCacheManagers() { - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(CacheMode.DIST_SYNC, true); - - //Verification for ISPN-2386 - the following exception should not appear: - //java.lang.ClassCastException: org.infinispan.marshall.MarshalledValue cannot be cast to org.infinispan.distexec.mapreduce.Book - - builder.eviction().maxEntries(1).strategy(EvictionStrategy.LRU); - builder.persistence().passivation(true).addStore(DummyInMemoryStoreConfigurationBuilder.class).storeName(getClass().getSimpleName()); - builder.storeAsBinary().enable(); - - createClusteredCaches(4, "bookSearch", builder); - } - -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/CompatModeDistributedTwoNodesMapReduceTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/CompatModeDistributedTwoNodesMapReduceTest.java deleted file mode 100644 index 8249aa8d5bf3..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/CompatModeDistributedTwoNodesMapReduceTest.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.Cache; -import org.infinispan.configuration.cache.CacheMode; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.infinispan.test.fwk.TestCacheManagerFactory; -import org.testng.annotations.Test; - -/** - * CompatModeDistributedTwoNodesMapReduceTest tests Map/Reduce functionality using two Infinispan nodes, - * distributed reduce and individual per task intermediate key/value cache, in compatibility mode. - * - * @author Martin Gencur - * @since 6.0 - */ -@Test(groups = "functional", testName = "distexec.mapreduce.CompatModeDistributedTwoNodesMapReduceTest") -public class CompatModeDistributedTwoNodesMapReduceTest extends BaseWordCountMapReduceTest { - - @Override - protected void createCacheManagers() throws Throwable { - ConfigurationBuilder builder = TestCacheManagerFactory.getDefaultCacheConfiguration(true, false); - builder.compatibility().enable() - .clustering() - .cacheMode(CacheMode.DIST_SYNC) - .stateTransfer().fetchInMemoryState(false) - .transaction().syncCommitPhase(true).syncRollbackPhase(true) - .cacheStopTimeout(0L); - createClusteredCaches(2, cacheName(), builder); - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - protected MapReduceTask createMapReduceTask(Cache c){ - //run distributed reduce with per task cache - return new MapReduceTask(c, true, false); - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/DistributedFourNodesMapReduceTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/DistributedFourNodesMapReduceTest.java deleted file mode 100644 index 55a6c4be6bcb..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/DistributedFourNodesMapReduceTest.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.Cache; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.testng.annotations.Test; - - -/** - * DistributedFourNodesMapReduceTest tests Map/Reduce functionality using four Infinispan nodes, - * distributed reduce and individual per task intermediate key/value cache - * - * @author Vladimir Blagojevic - * @since 5.2 - */ -@Test(groups = "functional", testName = "distexec.mapreduce.DistributedFourNodesMapReduceTest") -public class DistributedFourNodesMapReduceTest extends BaseWordCountMapReduceTest { - - @Override - protected void createCacheManagers() throws Throwable { - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(getCacheMode(), true); - builder.clustering().stateTransfer().chunkSize(2); - createClusteredCaches(4, cacheName(), builder); - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - protected MapReduceTask createMapReduceTask(Cache c){ - //run distributed reduce with per task cache - return new MapReduceTask(c, true, false); - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/DistributedIntermediateCacheFourNodesMapReduceTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/DistributedIntermediateCacheFourNodesMapReduceTest.java deleted file mode 100644 index 90687954fe40..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/DistributedIntermediateCacheFourNodesMapReduceTest.java +++ /dev/null @@ -1,68 +0,0 @@ -package org.infinispan.distexec.mapreduce; - - -import java.util.concurrent.TimeUnit; - -import org.infinispan.Cache; -import org.infinispan.commons.CacheException; -import org.infinispan.configuration.cache.CacheMode; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.testng.annotations.Test; - - -/** - * DistributedIntermediateCacheFourNodesMapReduceTest tests Map/Reduce functionality using four Infinispan nodes, - * distributed reduce and individual per task intermediate key/value cache specified by application - * - * @author Vladimir Blagojevic - * @since 7.0 - */ -@Test(groups = "functional", testName = "distexec.mapreduce.DistributedIntermediateCacheFourNodesMapReduceTest") -public class DistributedIntermediateCacheFourNodesMapReduceTest extends BaseWordCountMapReduceTest { - - private String intermediateCacheNameConfig = "tmpCacheConfig"; - - @Override - protected void createCacheManagers() throws Throwable { - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(getCacheMode(), true); - createClusteredCaches(4, cacheName(), builder); - } - - /** - * We are overriding createMapReduceTask factory method from superclass so that all test methods - * from superclass are run with application specified intermediate cache. - * - */ - @SuppressWarnings({ "rawtypes", "unchecked" }) - @Override - protected MapReduceTask createMapReduceTask(Cache c) { - //run distributed reduce with per task cache - cache specified by the user - MapReduceTask t = new MapReduceTask(c, true, - false); - ConfigurationBuilder cacheConfig = new ConfigurationBuilder(); - cacheConfig.unsafe().unreliableReturnValues(true) - .clustering().cacheMode(CacheMode.DIST_SYNC).hash().numOwners(2).sync(); - - //In the real world people will define configurations using xml files - defineConfigurationOnAllManagers(intermediateCacheNameConfig, cacheConfig); - return t.usingIntermediateCache(intermediateCacheNameConfig); - } - - @Test(expectedExceptions={CacheException.class}) - public void testIntermediateCacheNotCreatedOnAllNodes() throws Exception { - String cacheNameConfig = "notCreatedOnAllNodes"; - Cache c = cache(0, cacheName()); - MapReduceTask t = new MapReduceTask<>(c, true, false); - ConfigurationBuilder cacheConfig = new ConfigurationBuilder(); - cacheConfig.unsafe().unreliableReturnValues(true) - .clustering().cacheMode(CacheMode.DIST_SYNC).hash().numOwners(2) - .stateTransfer().timeout(5, TimeUnit.SECONDS); - - //define configuration only on first node - manager(0).defineConfiguration(cacheNameConfig, cacheConfig.build()); - - t.usingSharedIntermediateCache("irrelevant", cacheNameConfig); - t.mappedWith(new WordCountMapper()).reducedWith(new WordCountReducer()); - t.execute(); - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/DistributedIntermediateSharedCacheFourNodesMapReduceTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/DistributedIntermediateSharedCacheFourNodesMapReduceTest.java deleted file mode 100644 index b8865977227b..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/DistributedIntermediateSharedCacheFourNodesMapReduceTest.java +++ /dev/null @@ -1,68 +0,0 @@ -package org.infinispan.distexec.mapreduce; - - -import org.infinispan.Cache; -import org.infinispan.commons.CacheException; -import org.infinispan.configuration.cache.CacheMode; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.testng.annotations.Test; - -import java.util.concurrent.TimeUnit; - - -/** - * DistributedIntermediateSharedCacheFourNodesMapReduceTest tests Map/Reduce functionality using four Infinispan nodes, - * distributed reduce and shared intermediate key/value cache specified by application - * - * @author Vladimir Blagojevic - * @since 7.0 - */ -@Test(groups = "functional", testName = "distexec.mapreduce.DistributedIntermediateSharedCacheFourNodesMapReduceTest") -public class DistributedIntermediateSharedCacheFourNodesMapReduceTest extends BaseWordCountMapReduceTest { - - private String intermediateCacheName = "tmpCache"; - private String intermediateCacheNameConfig = "tmpCacheConfig"; - - @Override - protected void createCacheManagers() throws Throwable { - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(getCacheMode(), true); - createClusteredCaches(4, cacheName(), builder); - } - - /** - * We are overriding createMapReduceTask factory method from superclass so that all test methods - * from superclass are run with application specified intermediate cache. - * - */ - @SuppressWarnings({ "rawtypes", "unchecked" }) - @Override - protected MapReduceTask createMapReduceTask(Cache c) { - //run distributed reduce with per task cache - cache specified by the user - MapReduceTask t = new MapReduceTask<>(c, true, false); - ConfigurationBuilder cacheConfig = new ConfigurationBuilder(); - cacheConfig.unsafe().unreliableReturnValues(true) - .clustering().cacheMode(CacheMode.DIST_SYNC).hash().numOwners(2).sync(); - - //In the real world people will define configurations using xml files - defineConfigurationOnAllManagers(intermediateCacheNameConfig, cacheConfig); - return t.usingSharedIntermediateCache(intermediateCacheName, intermediateCacheNameConfig); - } - - @Test(expectedExceptions={CacheException.class}) - public void testIntermediateCacheNotCreatedOnAllNodes() throws Exception { - String cacheNameConfig = "notCreatedOnAllNodes"; - Cache c = cache(0, cacheName()); - MapReduceTask t = new MapReduceTask<>(c, true, false); - ConfigurationBuilder cacheConfig = new ConfigurationBuilder(); - cacheConfig.unsafe().unreliableReturnValues(true) - .clustering().cacheMode(CacheMode.DIST_SYNC).hash().numOwners(2) - .stateTransfer().timeout(5, TimeUnit.SECONDS); - - //define configuration only on first node - manager(0).defineConfiguration(cacheNameConfig, cacheConfig.build()); - - t.usingSharedIntermediateCache("irrelevant", cacheNameConfig); - t.mappedWith(new WordCountMapper()).reducedWith(new WordCountReducer()); - t.execute(); - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/DistributedSharedCacheFourNodesMapReduceTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/DistributedSharedCacheFourNodesMapReduceTest.java deleted file mode 100644 index 0005a2103090..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/DistributedSharedCacheFourNodesMapReduceTest.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.Cache; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.testng.annotations.Test; - -import java.util.Map; - -/** - * DistributedSharedCacheFourNodesMapReduceTest tests Map/Reduce functionality using four Infinispan nodes, - * distributed reduce and shared intermediate key/value cache - * - * @author Vladimir Blagojevic - * @since 5.2 - */ -@Test(groups = "functional", testName = "distexec.mapreduce.DistributedSharedCacheFourNodesMapReduceTest") -public class DistributedSharedCacheFourNodesMapReduceTest extends BaseWordCountMapReduceTest { - - @Override - protected void createCacheManagers() throws Throwable { - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(getCacheMode(), true); - createClusteredCaches(4, cacheName(), builder); - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - protected MapReduceTask createMapReduceTask(Cache c){ - //run distributed reduce with intermediate shared cache - return new MapReduceTask(c, true); - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/DistributedSharedCacheTwoNodesMapReduceTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/DistributedSharedCacheTwoNodesMapReduceTest.java deleted file mode 100644 index 2da3a313cc3b..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/DistributedSharedCacheTwoNodesMapReduceTest.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import static org.junit.Assert.assertEquals; - -import org.infinispan.Cache; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.infinispan.manager.EmbeddedCacheManager; -import org.testng.annotations.Test; - - -/** - * DistributedSharedCacheTwoNodesMapReduceTest tests Map/Reduce functionality using two Infinispan nodes, - * distributed reduce and shared intermediate key/value cache - * - * @author Vladimir Blagojevic - * @since 5.2 - */ -@Test(groups = "functional", testName = "distexec.mapreduce.DistributedSharedCacheTwoNodesMapReduceTest") -public class DistributedSharedCacheTwoNodesMapReduceTest extends BaseWordCountMapReduceTest { - - @Override - protected void createCacheManagers() throws Throwable { - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(getCacheMode(), true); - createClusteredCaches(2, cacheName(), builder); - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - protected MapReduceTask createMapReduceTask(Cache c){ - //run distributed reduce with intermediate shared cache - return new MapReduceTask(c, true); - } - - public MapReduceTask invokeMapReduce(String keys[]) throws Exception{ - MapReduceTask task = super.invokeMapReduce(keys, true); - - //Let's check that shared intermediate cache got cleaned up - //See https://issues.jboss.org/browse/ISPN-4471 for more details - EmbeddedCacheManager cacheManager = getCacheManagers().get(0); - String sharedCache = task.getIntermediateCacheName(); - assertEquals("MapReduceTask has not use shared intermediate cache as it should have but it used " + sharedCache, - MapReduceTask.DEFAULT_TMP_CACHE_CONFIGURATION_NAME, sharedCache); - Cache sharedTmpCache = cacheManager.getCache(sharedCache); - int elementsInCache = sharedTmpCache.size(); - assertEquals("Shared cache " + sharedCache + " is not empty. It has " + elementsInCache + " keys/values: " + - sharedTmpCache.entrySet(), 0, elementsInCache); - return task; - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/DistributedTwoNodesMapReduceTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/DistributedTwoNodesMapReduceTest.java deleted file mode 100644 index eadbf882eb8e..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/DistributedTwoNodesMapReduceTest.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.Cache; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.testng.annotations.Test; - -/** - * DistributedTwoNodesMapReduceTest tests Map/Reduce functionality using two Infinispan nodes, - * distributed reduce and individual per task intermediate key/value cache - * - * @author Vladimir Blagojevic - * @since 5.2 - */ -@Test(groups = "functional", testName = "distexec.mapreduce.DistributedTwoNodesMapReduceTest") -public class DistributedTwoNodesMapReduceTest extends BaseWordCountMapReduceTest { - - @Override - protected void createCacheManagers() throws Throwable { - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(getCacheMode(), true); - builder.clustering().stateTransfer().chunkSize(2); - createClusteredCaches(2, cacheName(), builder); - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - protected MapReduceTask createMapReduceTask(Cache c){ - //run distributed reduce with per task cache - return new MapReduceTask(c, true, false); - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedFourNodesMapReduceTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedFourNodesMapReduceTest.java deleted file mode 100644 index 466168692b22..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedFourNodesMapReduceTest.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import org.infinispan.Cache; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.testng.annotations.Test; - -/** - * LargeDistributedFourNodesMapReduceTest tests large Map/Reduce functionality using two Infinispan - * nodes, distributed reduce and individual per task intermediate key/value cache - * - * @author Vladimir Blagojevic - * @since 7.0 - */ -@Test(groups = "stress", testName = "distexec.mapreduce.LargeDistributedFourNodesMapReduceTest") -public class LargeDistributedFourNodesMapReduceTest extends BaseLargeWordCountMapReduceTest { - - @Override - protected void createCacheManagers() throws Throwable { - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(getCacheMode(), true); - builder.clustering().stateTransfer().sync().replTimeout(45, TimeUnit.SECONDS); - createClusteredCaches(4, cacheName(), builder); - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - protected MapReduceTask createMapReduceTask(Cache c) { - //run distributed reduce with per task cache - return new MapReduceTask(c, true, false); - } - - @Override - public void testInvokeMapReduceOnAllKeys() throws Exception { - MapReduceTask task = invokeMapReduce(null); - log.debug("Read macbeth.txt and inserted keys into cache. Executing M/R task..."); - long start = System.currentTimeMillis(); - Map mapReduce = task.execute(); - log.debugf("Task completed in %s ms", System.currentTimeMillis() - start); - verifyResults(mapReduce); - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedFourNodesMapReduceWithEvictionTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedFourNodesMapReduceWithEvictionTest.java deleted file mode 100644 index 9a2dbc7fa2eb..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedFourNodesMapReduceWithEvictionTest.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import java.util.concurrent.TimeUnit; - -import org.infinispan.Cache; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.testng.annotations.Test; - -/** - * LargeDistributedFourNodesMapReduceWithEvictionTest tests large Map/Reduce functionality using - * four Infinispan nodes, distributed reduce and individual per task intermediate key/value cache. - * - * Eviction is enabled in order to turn on BCHM backed DataContainer. - * - * @author Vladimir Blagojevic - * @since 7.0 - */ -@Test(groups = "stress", testName = "distexec.mapreduce.LargeDistributedFourNodesMapReduceWithEvictionTest") -public class LargeDistributedFourNodesMapReduceWithEvictionTest extends BaseLargeWordCountMapReduceTest { - - @Override - protected void createCacheManagers() throws Throwable { - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(getCacheMode(), true); - builder.clustering().stateTransfer().sync().replTimeout(45, TimeUnit.SECONDS).eviction() - .maxEntries(100000); - createClusteredCaches(4, cacheName(), builder); - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - protected MapReduceTask createMapReduceTask(Cache c) { - //run distributed reduce with per task cache - return new MapReduceTask(c, true, false); - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedSharedCacheFourNodesMapReduceTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedSharedCacheFourNodesMapReduceTest.java deleted file mode 100644 index 9b98458db20f..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedSharedCacheFourNodesMapReduceTest.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import java.util.concurrent.TimeUnit; - -import org.infinispan.Cache; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.testng.annotations.Test; - - -/** - * LargeDistributedSharedCacheFourNodesMapReduceTest tests Map/Reduce functionality using four Infinispan nodes, - * distributed reduce and shared intermediate key/value cache - * - * @author Vladimir Blagojevic - * @since 7.0 - */ -@Test(groups = "stress", testName = "distexec.mapreduce.LargeDistributedSharedCacheFourNodesMapReduceTest") -public class LargeDistributedSharedCacheFourNodesMapReduceTest extends BaseLargeWordCountMapReduceTest { - - @Override - protected void createCacheManagers() throws Throwable { - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(getCacheMode(), true); - builder.clustering().stateTransfer().sync().replTimeout(45, TimeUnit.SECONDS); - createClusteredCaches(4, cacheName(), builder); - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - protected MapReduceTask createMapReduceTask(Cache c){ - //run distributed reduce with intermediate shared cache - return new MapReduceTask(c, true); - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedSharedCacheFourNodesMapReduceWithEvictionTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedSharedCacheFourNodesMapReduceWithEvictionTest.java deleted file mode 100644 index 7c0ae566a957..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedSharedCacheFourNodesMapReduceWithEvictionTest.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import java.util.concurrent.TimeUnit; - -import org.infinispan.Cache; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.testng.annotations.Test; - - -/** - * Eviction is enabled in order to turn on BCHM backed DataContainer tests Map/Reduce functionality using - * four Infinispan nodes, distributed reduce and shared intermediate key/value cache. - * - * Eviction is enabled in order to turn on BCHM backed DataContainer. - * - * @author Vladimir Blagojevic - * @since 7.0 - */ -@Test(groups = "stress", testName = "distexec.mapreduce.LargeDistributedSharedCacheFourNodesMapReduceWithEvictionTest") -public class LargeDistributedSharedCacheFourNodesMapReduceWithEvictionTest extends BaseLargeWordCountMapReduceTest { - - @Override - protected void createCacheManagers() throws Throwable { - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(getCacheMode(), true); - builder.clustering().stateTransfer().sync().replTimeout(45, TimeUnit.SECONDS).eviction() - .maxEntries(100000); - createClusteredCaches(4, cacheName(), builder); - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - protected MapReduceTask createMapReduceTask(Cache c){ - //run distributed reduce with intermediate shared cache - return new MapReduceTask(c, true); - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedSharedCacheTwoNodesMapReduceTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedSharedCacheTwoNodesMapReduceTest.java deleted file mode 100644 index 0addead1c081..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedSharedCacheTwoNodesMapReduceTest.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import java.util.concurrent.TimeUnit; - -import org.infinispan.Cache; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.testng.annotations.Test; - -/** - * LargeDistributedSharedCacheTwoNodesMapReduceTest tests large Map/Reduce functionality using two - * Infinispan nodes, distributed reduce and shared intermediate key/value cache - * - * @author Vladimir Blagojevic - * @since 7.0 - */ -@Test(groups = "stress", testName = "distexec.mapreduce.LargeDistributedSharedCacheTwoNodesMapReduceTest") -public class LargeDistributedSharedCacheTwoNodesMapReduceTest extends BaseLargeWordCountMapReduceTest { - - @Override - protected void createCacheManagers() throws Throwable { - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(getCacheMode(), true); - builder.clustering().stateTransfer().sync().replTimeout(45, TimeUnit.SECONDS); - createClusteredCaches(2, cacheName(), builder); - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - protected MapReduceTask createMapReduceTask(Cache c){ - //run distributed reduce with intermediate shared cache - return new MapReduceTask(c, true); - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedTwoNodesMapReduceTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedTwoNodesMapReduceTest.java deleted file mode 100644 index 47e4847d5a04..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/LargeDistributedTwoNodesMapReduceTest.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import java.util.concurrent.TimeUnit; - -import org.infinispan.Cache; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.testng.annotations.Test; - -/** - * LargeDistributedTwoNodesMapReduceTest tests large Map/Reduce functionality using two Infinispan - * nodes, distributed reduce and individual per task intermediate key/value cache - * - * @author Vladimir Blagojevic - * @since 7.0 - */ -@Test(groups = "stress", testName = "distexec.mapreduce.LargeDistributedTwoNodesMapReduceTest") -public class LargeDistributedTwoNodesMapReduceTest extends BaseLargeWordCountMapReduceTest { - - @Override - protected void createCacheManagers() throws Throwable { - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(getCacheMode(), true); - builder.clustering().stateTransfer().sync().replTimeout(45, TimeUnit.SECONDS); - createClusteredCaches(2, cacheName(), builder); - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - protected MapReduceTask createMapReduceTask(Cache c) { - //run distributed reduce with per task cache - return new MapReduceTask(c, true, false); - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/LocalMapReduceTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/LocalMapReduceTest.java deleted file mode 100644 index 66434ec76819..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/LocalMapReduceTest.java +++ /dev/null @@ -1,165 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.infinispan.test.TestingUtil.now; -import static org.testng.AssertJUnit.assertEquals; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.infinispan.Cache; -import org.infinispan.configuration.cache.CacheMode; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.infinispan.manager.EmbeddedCacheManager; -import org.infinispan.test.fwk.TestCacheManagerFactory; -import org.testng.annotations.Test; - -/** - * ReplicatedFourNodesMapReduceTest tests Map/Reduce functionality using four Infinispan nodes, - * replicated reduce and individual per task intermediate key/value cache - * - * @author William Burns - * @since 5.3 - */ -@Test(groups = "functional", testName = "distexec.mapreduce.LocalMapReduceTest") -public class LocalMapReduceTest extends DistributedFourNodesMapReduceTest { - - public static final int EXPIRATION_TIMEOUT = 3000; - public static final int EVICTION_CHECK_TIMEOUT = 2000; - - @Override - protected CacheMode getCacheMode() { - return CacheMode.LOCAL; - } - - @Override - protected void createCacheManagers() throws Throwable { - EmbeddedCacheManager cacheManager = TestCacheManagerFactory.createCacheManager(false); - cacheManagers.add(cacheManager); - } - - @Override - protected MapReduceTask createMapReduceTask(Cache c){ - return new MapReduceTask(c, false, false); - } - - @Override - public void testInvokeMapReduceOnSubsetOfKeysWithResultCache() throws Exception { - String cacheName = "resultCache2"; - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(CacheMode.LOCAL, true); - defineConfigurationOnAllManagers(cacheName, builder); - try { - MapReduceTask task = invokeMapReduce(new String[] { "1", "2", "3" }); - task.execute(cacheName); - Cache c1 = cache(0, cacheName); - assertPartialWordCount(countWords(c1)); - c1.clear(); - } finally { - removeCacheFromCluster(cacheName); - } - } - - @Override - public void testInvokeMapReduceOnAllKeysWithResultCache() throws Exception { - String cacheName = "resultCache"; - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(CacheMode.LOCAL, true); - defineConfigurationOnAllManagers(cacheName, builder); - try { - MapReduceTask task = invokeMapReduce(null); - Cache c1 = cache(0, cacheName); - task.execute(c1); - verifyResults(c1); - c1.clear(); - } finally { - removeCacheFromCluster(cacheName); - } - } - - public void testFilterExpiredInvokingMap() throws Exception { - String cacheName = "expiredResultCache"; - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(CacheMode.LOCAL, true); - defineConfigurationOnAllManagers(cacheName, builder); - Cache c1 = cache(0, cacheName()); - try { - final long lifespan = EXPIRATION_TIMEOUT; - final long startTime = now(); - Map dataIn = getDataIn(); - Collection valuesIn = dataIn.values(); - c1.putAll(dataIn, lifespan, MILLISECONDS); - expectCachedAndThenExpired(c1, valuesIn, startTime, lifespan); - WordCountMapper mapper = new WordCountMapper(); - WordCountReducer reducer = new WordCountReducer(); - MapReduceTask task = createMapReduceTask(c1); - task.mappedWith(mapper).reducedWith(reducer); - Map result = task.execute(); - assertEquals(0, result.size()); - } finally { - c1.clear(); - removeCacheFromCluster(cacheName); - } - } - - private void expectCachedAndThenExpired(Cache cache, Collection valuesIn, long startTime, long lifespan) throws Exception { - List values; - while (true) { - values = new ArrayList<>(cache.values()); - if (now() >= startTime + lifespan) - break; - expectUnorderedEquals(valuesIn, values); - Thread.sleep(100); - } - - while (now() < startTime + lifespan + EVICTION_CHECK_TIMEOUT) { - values = new ArrayList<>(cache.values()); - if (values.size() == 0) return; - } - - assertEquals(0, values.size()); - } - - public Map getDataIn() { - Map dataIn = new HashMap(); - dataIn.put("1", "Hello world here I am"); - dataIn.put("2", "Infinispan rules the world"); - dataIn.put("3", "JUDCon is in Boston"); - dataIn.put("4", "JBoss World is in Boston as well"); - dataIn.put("12","JBoss Application Server"); - dataIn.put("15", "Hello world"); - dataIn.put("14", "Infinispan community"); - dataIn.put("111", "Infinispan open source"); - dataIn.put("112", "Boston is close to Toronto"); - dataIn.put("113", "Toronto is a capital of Ontario"); - dataIn.put("114", "JUDCon is cool"); - dataIn.put("211", "JBoss World is awesome"); - dataIn.put("212", "JBoss rules"); - dataIn.put("213", "JBoss division of RedHat "); - dataIn.put("214", "RedHat community"); - return dataIn; - } - - private boolean expectUnorderedEquals(Collection h1, Collection h2) { - if (h1.size() != h2.size()) - return false; - - List clone = new ArrayList<>(h2); - for (T next : h1) { - if (clone.contains(next)) - clone.remove(next); - else - return false; - } - return true; - } - - @Override - /** - * Method is overridden so that there is 1 cache for what the test may think are different managers - * since local only has 1 manager. - */ - protected Cache cache(int index, String cacheName) { - return super.cache(0, cacheName); - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/LocalSimpleBookSearchTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/LocalSimpleBookSearchTest.java deleted file mode 100644 index f9d2b82031b5..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/LocalSimpleBookSearchTest.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.Cache; -import org.infinispan.configuration.cache.CacheMode; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.testng.annotations.Test; - -/** - * Tests the mapreduce with simple configuration with a local cache with a transport. - * - * @author William Burns - * @since 5.3 - */ -@Test(groups = "functional", testName = "distexec.mapreduce.LocalSimpleBookSearchTest") -public class LocalSimpleBookSearchTest extends BookSearchTest { - - @Override - protected void createCacheManagers() { - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(CacheMode.LOCAL, true); - createClusteredCaches(1, "bookSearch" ,builder); - } - - @Override - /** - * Method is overridden so that there is 1 cache for what the test may think are different managers - * since local only has 1 manager. - */ - protected Cache cache(int index, String cacheName) { - return super.cache(0, cacheName); - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/ReplicatedFourNodesMapReduceTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/ReplicatedFourNodesMapReduceTest.java deleted file mode 100644 index 7981e9877e2c..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/ReplicatedFourNodesMapReduceTest.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.Cache; -import org.infinispan.configuration.cache.CacheMode; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.testng.annotations.Test; - -/** - * ReplicatedFourNodesMapReduceTest tests Map/Reduce functionality using four Infinispan nodes, - * replicated reduce and individual per task intermediate key/value cache - * - * @author wburns - * @since 5.3 - */ -@Test(groups = "functional", testName = "distexec.mapreduce.ReplicatedFourNodesMapReduceTest") -public class ReplicatedFourNodesMapReduceTest extends DistributedFourNodesMapReduceTest { - - @Override - protected CacheMode getCacheMode() { - return CacheMode.REPL_SYNC; - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/ReplicatedTwoNodesMapReduceTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/ReplicatedTwoNodesMapReduceTest.java deleted file mode 100644 index 5fbed86e2ee8..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/ReplicatedTwoNodesMapReduceTest.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.Cache; -import org.infinispan.configuration.cache.CacheMode; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.testng.annotations.Test; - -/** - * ReplicatedFourNodesMapReduceTest tests Map/Reduce functionality using four Infinispan nodes, - * replicated reduce and individual per task intermediate key/value cache - * - * @author William Burns - * @since 5.3 - */ -@Test(groups = "functional", testName = "distexec.mapreduce.ReplicatedTwoNodesMapReduceTest") -public class ReplicatedTwoNodesMapReduceTest extends DistributedTwoNodesMapReduceTest { - - @Override - protected CacheMode getCacheMode() { - return CacheMode.REPL_SYNC; - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/SimpleBookSearchTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/SimpleBookSearchTest.java deleted file mode 100644 index 9d8743f044fe..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/SimpleBookSearchTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.configuration.cache.CacheMode; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.testng.annotations.Test; - -/** - * Tests the mapreduce with simple configuration. - * - * @author Anna Manukyan - */ -@Test(groups = "functional", testName = "distexec.mapreduce.SimpleBookSearchTest") -public class SimpleBookSearchTest extends BookSearchTest { - - @Override - protected void createCacheManagers() { - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(CacheMode.DIST_SYNC, true); - createClusteredCaches(4, "bookSearch" ,builder); - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/SimpleFourNodesMapReduceTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/SimpleFourNodesMapReduceTest.java deleted file mode 100644 index bb9c3f9d3f27..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/SimpleFourNodesMapReduceTest.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.testng.annotations.Test; - -/** - * SimpleFourNodesMapReduceTest tests Map/Reduce functionality using four Infinispan nodes and local - * reduce - * - * @author Vladimir Blagojevic - * @since 5.0 - */ -@Test(groups = "functional", testName = "distexec.mapreduce.SimpleFourNodesMapReduceTest") -public class SimpleFourNodesMapReduceTest extends BaseWordCountMapReduceTest { - - @Override - protected void createCacheManagers() throws Throwable { - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(getCacheMode(), true); - createClusteredCaches(4, cacheName(), builder); - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/SimpleMapReduceTaskTimeoutTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/SimpleMapReduceTaskTimeoutTest.java deleted file mode 100644 index aed7e3830984..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/SimpleMapReduceTaskTimeoutTest.java +++ /dev/null @@ -1,241 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.Cache; -import org.infinispan.commons.CacheException; -import org.infinispan.configuration.cache.CacheMode; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.infinispan.test.MultipleCacheManagersTest; -import org.infinispan.util.concurrent.TimeoutException; -import org.testng.annotations.Test; - -import java.util.Iterator; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.NANOSECONDS; -import static org.infinispan.distribution.DistributionTestHelper.isFirstOwner; -import static org.testng.AssertJUnit.assertEquals; -import static org.testng.AssertJUnit.assertTrue; -import static org.testng.AssertJUnit.fail; - -/** - * @author Pedro Ruivo - * @since 5.3 - */ -@Test(groups = "functional", testName = "distexec.mapreduce.SimpleMapReduceTaskTimeoutTest") -public class SimpleMapReduceTaskTimeoutTest extends MultipleCacheManagersTest { - - private static final int REPLICATION_TIMEOUT = 5000; - private static final int DEFAULT_TIMEOUT = 0; - - public SimpleMapReduceTaskTimeoutTest() { - this.cleanup = CleanupPhase.AFTER_METHOD; - } - - /** - * Tests a map/reduce task with duration between replication timeout and task timeout (no exception should be - * thrown) - */ - public void testTimeout() { - final int taskTimeout = REPLICATION_TIMEOUT * 4; - final int sleepTime = REPLICATION_TIMEOUT * 2; - final String sleepOnKey = init(); - - MapReduceTask task = createMapReduceTask(this.cache(0)); - assertEquals("Wrong task timeout.", DEFAULT_TIMEOUT, task.timeout(MILLISECONDS)); - assertEquals("Wrong replication timeout.", REPLICATION_TIMEOUT, - cache(0).getCacheConfiguration().clustering().sync().replTimeout()); - task.timeout(taskTimeout, MILLISECONDS); - assertEquals("Wrong new task timeout.", taskTimeout, task.timeout(MILLISECONDS)); - - task.mappedWith(new SleepMapper(sleepTime, sleepOnKey)) - .reducedWith(new DummyReducer()); - - long start = System.nanoTime(); - task.execute(); - long duration = System.nanoTime() - start; - assertTrue(NANOSECONDS.toMillis(duration) >= sleepTime); - } - - /** - * Tests a map/reduce task with duration between task timeout and replication timeout (exception expected!) - */ - public void testTimeout2() throws Exception { - testTimeoutHelper(true); - } - - /** - * Tests async map/reduce task with duration between task timeout and replication timeout (exception expected!) - */ - public void testTimeoutAsync() throws Exception { - testTimeoutHelper(false); - } - - /** - * Tests async map/reduce task with duration between task timeout and replication timeout (exception expected!) - */ - private void testTimeoutHelper(boolean sync) throws Exception { - final int taskTimeout = REPLICATION_TIMEOUT / 4; - final int sleepTime = REPLICATION_TIMEOUT / 2; - final String sleepOnKey = init(); - - MapReduceTask task = createMapReduceTask(this. cache(0)); - assertEquals("Wrong task timeout.", DEFAULT_TIMEOUT, task.timeout(MILLISECONDS)); - assertEquals("Wrong replication timeout.", REPLICATION_TIMEOUT, cache(0).getCacheConfiguration().clustering() - .sync().replTimeout()); - task.timeout(taskTimeout, MILLISECONDS); - assertEquals("Wrong new task timeout.", taskTimeout, task.timeout(MILLISECONDS)); - - task.mappedWith(new SleepMapper(sleepTime, sleepOnKey)).reducedWith(new DummyReducer()); - - long start = System.nanoTime(); - if (sync) { - try { - task.execute(); - fail("Should have gotten an exception for task.execute() call"); - } catch (CacheException expected) { - assertTrue(hasTimeoutException(expected)); - } - } else { - try { - Future> future = task.executeAsynchronously(); - future.get(); - fail("Should have gotten an exception for future.get() call"); - } catch (ExecutionException e) { - assertTrue(hasTimeoutException(e)); - } - } - long duration = System.nanoTime() - start; - assertTrue(NANOSECONDS.toMillis(duration) >= taskTimeout); - } - - /** - * Tests a map/reduce task with duration higher than replication timeout and waiting forever (no exception should be - * thrown) - */ - public void testNegativeTimeout() { - final int taskTimeout = -1; - final int sleepTime = REPLICATION_TIMEOUT * 2; //higher thant the replication timeout - final String sleepOnKey = init(); - - MapReduceTask task = createMapReduceTask(this.cache(0)); - assertEquals("Wrong task timeout.", DEFAULT_TIMEOUT, task.timeout(MILLISECONDS)); - assertEquals("Wrong replication timeout.", REPLICATION_TIMEOUT, - cache(0).getCacheConfiguration().clustering().sync().replTimeout()); - task.timeout(taskTimeout, MILLISECONDS); - assertEquals("Wrong new task timeout.", taskTimeout, task.timeout(MILLISECONDS)); - - task.mappedWith(new SleepMapper(sleepTime, sleepOnKey)) - .reducedWith(new DummyReducer()); - - long start = System.nanoTime(); - task.execute(); - long duration = System.nanoTime() - start; - assertTrue(NANOSECONDS.toMillis(duration) >= sleepTime); - } - - /** - * Tests a map/reduce task with duration higher than replication timeout and waiting forever (no exception should be - * thrown) - */ - public void testZeroTimeout() { - final int taskTimeout = 0; - final int sleepTime = REPLICATION_TIMEOUT * 2; //higher thant the replication timeout - final String sleepOnKey = init(); - - MapReduceTask task = createMapReduceTask(this.cache(0)); - assertEquals("Wrong task timeout.", DEFAULT_TIMEOUT, task.timeout(MILLISECONDS)); - assertEquals("Wrong replication timeout.", REPLICATION_TIMEOUT, - cache(0).getCacheConfiguration().clustering().sync().replTimeout()); - task.timeout(taskTimeout, MILLISECONDS); - assertEquals("Wrong new task timeout.", taskTimeout, task.timeout(MILLISECONDS)); - - task.mappedWith(new SleepMapper(sleepTime, sleepOnKey)) - .reducedWith(new DummyReducer()); - - long start = System.nanoTime(); - task.execute(); - long duration = System.nanoTime() - start; - assertTrue(NANOSECONDS.toMillis(duration) >= sleepTime); - } - - @Override - protected void createCacheManagers() throws Throwable { - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(CacheMode.DIST_SYNC, true); - builder.clustering().sync().replTimeout(REPLICATION_TIMEOUT); - createClusteredCaches(4, builder); - } - - protected MapReduceTask createMapReduceTask(Cache c) { - return new MapReduceTask(c); - } - - private boolean hasTimeoutException(Exception exception) { - Throwable iterator = exception; - while (iterator != null) { - if (iterator instanceof TimeoutException) { - return true; - } - iterator = iterator.getCause(); - } - return false; - } - - /** - * Initializes all the caches with some data. - * - * @return a key owned by cache(1). - */ - private String init() { - String sleepOnKey = null; - for (int i = 0; i < 100; ++i) { - String key = String.valueOf(i); - cache(0).put(key, "v" + (i % 4)); - if (sleepOnKey == null && isFirstOwner(cache(1), key)) { - sleepOnKey = key; - } - } - return sleepOnKey; - } - - public static class SleepMapper implements Mapper { - - private final long sleepTime; - private final String sleepOnKey; - private boolean alreadySlept; - - public SleepMapper(long sleepTime, String sleepOnKey) { - this.sleepTime = sleepTime; - this.sleepOnKey = sleepOnKey; - this.alreadySlept = false; - } - - @Override - public void map(String key, String value, Collector collector) { - if (!alreadySlept && key.equals(sleepOnKey)) { - try { - Thread.sleep(sleepTime); - } catch (InterruptedException e) { - //no-op - Thread.currentThread().interrupt(); - } - alreadySlept = true; - } - collector.emit(value, 1); - } - } - - public static class DummyReducer implements Reducer { - - @Override - public Integer reduce(String reducedKey, Iterator iter) { - int sum = 0; - while (iter.hasNext()) { - sum += iter.next(); - } - return sum; - } - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/SimpleTwoNodesMapReduceTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/SimpleTwoNodesMapReduceTest.java deleted file mode 100644 index 050ffb71a6c4..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/SimpleTwoNodesMapReduceTest.java +++ /dev/null @@ -1,367 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.Cache; -import org.infinispan.commons.CacheException; -import org.infinispan.commons.util.CollectionFactory; -import org.infinispan.configuration.cache.CacheMode; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.infinispan.test.CacheManagerCallable; -import org.infinispan.test.fwk.TestCacheManagerFactory; -import org.infinispan.util.logging.Log; -import org.infinispan.util.logging.LogFactory; -import org.testng.annotations.Test; - -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.CancellationException; -import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static org.infinispan.test.TestingUtil.withCacheManager; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -/** - * SimpleTwoNodesMapReduceTest tests Map/Reduce functionality using two Infinispan nodes and local - * reduce - * - * @author Vladimir Blagojevic - * @since 5.0 - */ -@Test(groups = "functional", testName = "distexec.mapreduce.SimpleTwoNodesMapReduceTest") -public class SimpleTwoNodesMapReduceTest extends BaseWordCountMapReduceTest { - - private static final Log log = LogFactory.getLog(SimpleTwoNodesMapReduceTest.class); - protected static final Map barriers = CollectionFactory.makeConcurrentMap(); - - @Override - protected void createCacheManagers() throws Throwable { - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(getCacheMode(), true); - createClusteredCaches(2, cacheName(), builder); - } - - @Test(expectedExceptions={ExecutionException.class}) - public void testInvokeMapperCancellation() throws Exception { - final CyclicBarrier barrier = new CyclicBarrier(nodeCount() + 1); - final String name = this.getClass().getSimpleName(); - barriers.put(name, barrier); - - MapReduceTask task = invokeMapReduce(null, - new LatchMapper(name), new WordCountReducer()); - final Future> future = task.executeAsynchronously(); - Future cancelled = fork(new Callable() { - - @Override - public Boolean call() throws Exception { - //make sure that all nodes receive the command and... - barrier.await(10, TimeUnit.SECONDS); - - //...are ready to be canceled - return future.cancel(true); - } - }); - boolean mapperCancelled = false; - Throwable root = null; - try { - future.get(); - } catch (Exception e) { - root = e; - while(root.getCause() != null){ - root = root.getCause(); - } - mapperCancelled = root.getClass().equals(InterruptedException.class); - } - assertTrue("Mapper not cancelled, root cause " + root, mapperCancelled); - assertTrue(cancelled.get()); - assertTrue(future.isDone()); - - //Cancelling again - should return false - boolean canceled = future.cancel(true); - assertFalse(canceled); - //now call get() again and it should throw CancellationException - future.get(); - } - - static class LatchMapper implements Mapper { - - private static final long serialVersionUID = 2518908878377582179L; - private final String name; - - LatchMapper(String name) { - this.name = name; - } - - @Override - public void map(String key, String value, Collector collector) { - CyclicBarrier barrier = barriers.get(name); - - try { - barrier.await(10, TimeUnit.SECONDS); - TimeUnit.MILLISECONDS.sleep(5000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } catch (Exception e) { - log.error("Error in the mapping phase", e); - } - } - } - - @Test(expectedExceptions = IllegalArgumentException.class) - public void testInvokeMapReduceNullMapper() throws Exception { - invokeMapReduce(null, null, null); - } - - @Test(expectedExceptions = IllegalArgumentException.class) - public void testInvokeMapReduceNullReducer() throws Exception { - invokeMapReduce(null, new ExceptionMapper(false), null); - } - - @Test(expectedExceptions = IllegalArgumentException.class) - public void testInvokeMapReduceNullCombiner() throws Exception { - Cache cacheObj = cache(0, cacheName()); - MapReduceTask task = new MapReduceTask(cacheObj); - MapReduceTask task1 = new MapReduceTask(cacheObj); - - task.mappedWith(new ExceptionMapper(false)).reducedWith(new ExceptionReducer(false)); - task.combinedWith(null); - } - - @Test(expectedExceptions = IllegalArgumentException.class) - public void testInvokeMapReduceWithNullMasterCache() { - createMapReduceTask(null); - } - - @Test(expectedExceptions = CacheException.class) - public void testInvokeMapReduceWithException() throws Exception { - MapReduceTask task = invokeMapReduce(null, new ExceptionMapper(true), new ExceptionReducer(false)); - task.execute(); - } - - @Test(expectedExceptions = CacheException.class) - public void testInvokeMapWithReduceExceptionPhaseInLocalExecution() throws Exception { - Cache cache1 = cache(0, cacheName()); - Cache cache2 = cache(1, cacheName()); - - cache1.put("key1", "value1"); - cache2.put("key2", "valu2"); - cache2.put("key3", "valu2"); - cache2.put("key4", "valu2"); - cache2.put("key5", "valu2"); - MapReduceTask task = new MapReduceTask(cache1, true, false); - task.mappedWith(new WordCountMapper()). - reducedWith(new ExceptionReducer(true)); - - Map val = task.execute(); - } - - @Test(expectedExceptions = CacheException.class) - public void testInvokeMapWithReduceExceptionPhaseInRemoteExecution() throws Exception { - Cache cache1 = cache(0, cacheName()); - Cache cache2 = cache(1, cacheName()); - - cache1.put("key1", "value1"); - cache2.put("key2", "valu2"); - cache2.put("key3", "valu2"); - cache2.put("key4", "valu2"); - cache2.put("key5", "valu2"); - MapReduceTask task = new MapReduceTask(cache1, true, false); - task.mappedWith(new WordCountMapper()). - reducedWith(new FailAfterSecondCallReducer()); - - Map val = task.execute(); - } - - public void testMapReduceTasksComparison() throws Exception { - MapReduceTask task = invokeMapReduce(null, new ExceptionMapper(false), new ExceptionReducer(false)); - MapReduceTask task1 = invokeMapReduce(null, new ExceptionMapper(false), new ExceptionReducer(false)); - MapReduceTask task2 = invokeMapReduce(null, new ExceptionMapper(false), new ExceptionReducer(false)); - MapReduceTask task3 = null; - MapReduceTask task4 = task1; - - Object objectForCompare = new Object(); - - Map taskMap = new HashMap(); - taskMap.put(task, 1); - taskMap.put(task1, 2); - - - assertNotNull(taskMap.get(task)); - assertNotNull(taskMap.get(task1)); - assertNull(taskMap.get(task2)); - - assertFalse(task1.equals(task2)); - assertFalse(task1.equals(task3)); - assertFalse(task1.equals(objectForCompare)); - assertTrue(task1.equals(task4)); - - Pattern p = Pattern.compile("MapReduceTask \\[mapper=\\S+, reducer=\\S+, combiner=\\S*, keys=\\S*, taskId=\\S+\\]"); - Matcher m = p.matcher(task1.toString()); - assertTrue(m.find()); - } - - @Test(expectedExceptions = ExecutionException.class) - public void testInvokeAsynchronouslyWithException() throws Exception { - MapReduceTask task = invokeMapReduce(null, new ExceptionMapper(true), new ExceptionReducer(true)); - - Future> futureMap = task.executeAsynchronously(); - Map resultMap = futureMap.get(); - } - - @Test(expectedExceptions = ExecutionException.class) - public void testInvokeAsynchronouslyWithCollatorAndException() throws Exception { - MapReduceTask task = invokeMapReduce(null, new ExceptionMapper(true), new ExceptionReducer(true)); - - Future futureMap = task.executeAsynchronously(new Collator() { - - @Override - public Integer collate(Map reducedResults) { - return 0; - } - }); - - futureMap.get(); - } - - @Test(expectedExceptions = IllegalStateException.class) - public void testEnsureProperCacheState() throws Exception { - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(CacheMode.DIST_SYNC, true); - withCacheManager(new CacheManagerCallable(TestCacheManagerFactory.createClusteredCacheManager(builder) - ) { - @Override - public void call() { - try { - Cache cache = cm.getCache(); - cache.stop(); - - MapReduceTask task = createMapReduceTask(cache); - } catch(IllegalStateException ex) { - assertNotNull(ex.getMessage()); - assertTrue(ex.getMessage().contains("Cache is in an invalid state:")); - throw ex; - } - } - }); - } - - @Test(expectedExceptions = IllegalStateException.class) - public void testEnsureProperCacheStateMode() { - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(CacheMode.INVALIDATION_SYNC, true); - - withCacheManager(new CacheManagerCallable(TestCacheManagerFactory.createClusteredCacheManager(builder)) { - - @SuppressWarnings("unused") - @Override - public void call() { - Cache cache = cm.getCache(); - MapReduceTask task = new MapReduceTask( - cache); - } - }); - } - - @Test(expectedExceptions = CacheException.class) - public void testCombinerWithException() throws Exception { - MapReduceTask task = invokeMapReduce(null); - task.combinedWith(new ExceptionReducer(true)); - - task.execute(); - } - - public void testIntermediateCompositeKeys() { - MapReduceManagerImpl.IntermediateKey key = new MapReduceManagerImpl.IntermediateKey("task1", 1); - MapReduceManagerImpl.IntermediateKey key1 = new MapReduceManagerImpl.IntermediateKey("task2", 2); - MapReduceManagerImpl.IntermediateKey key2 = new MapReduceManagerImpl.IntermediateKey("task1", 1); - - //Composite keys with null taskID - MapReduceManagerImpl.IntermediateKey key3 = new MapReduceManagerImpl.IntermediateKey(null, 1); - MapReduceManagerImpl.IntermediateKey key4 = new MapReduceManagerImpl.IntermediateKey(null, 2); - - //Composite keys with null keys - MapReduceManagerImpl.IntermediateKey key5 = new MapReduceManagerImpl.IntermediateKey("task3", null); - MapReduceManagerImpl.IntermediateKey key6 = new MapReduceManagerImpl.IntermediateKey("task4", null); - - //Composite keys with null keys & taskIDs - MapReduceManagerImpl.IntermediateKey key7 = new MapReduceManagerImpl.IntermediateKey(null, null); - MapReduceManagerImpl.IntermediateKey key8 = new MapReduceManagerImpl.IntermediateKey(null, null); - - assertFalse(key.equals(key1)); - assertTrue(key.equals(key2)); - assertFalse(key.equals(null)); - assertFalse(key.equals(new String())); - - assertFalse(key3.equals(key4)); - assertFalse(key5.equals(key6)); - - assertTrue(key7.equals(key8)); - - assertFalse(key3.equals(key)); - assertFalse(key5.equals(key)); - - } - - private static class ExceptionMapper implements Mapper { - /** The serialVersionUID */ - private static final long serialVersionUID = -5943370243108735560L; - private boolean throwException = false; - - public ExceptionMapper(boolean throwException) { - this.throwException = throwException; - } - - @Override - public void map(String key, String value, Collector collector) { - if(value == null) throw new IllegalArgumentException("Key " + key + " has value " + value); - - if(throwException) { - //simulating exception here - int a = 4 / 0; - } - } - } - - private static class ExceptionReducer implements Reducer { - /** The serialVersionUID */ - private static final long serialVersionUID = 1901016598354633256L; - private boolean throwException; - - public ExceptionReducer(boolean throwException) { - this.throwException = throwException; - } - - @Override - public Integer reduce(String key, Iterator iter) { - if(throwException) { - //simulating exception - int a = 4 / 0; - } - - return 0; - } - } - - private static class FailAfterSecondCallReducer implements Reducer { - /** The serialVersionUID */ - private static final long serialVersionUID = 1901016598354633256L; - private static int counter; - - @Override - public Integer reduce(String key, Iterator iter) { - - if(counter > 0) { - //simulating exception - int a = 4 / 0; - } - counter++; - - return 0; - } - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/TopologyAwareTwoNodesMapReduceTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/TopologyAwareTwoNodesMapReduceTest.java deleted file mode 100644 index 0ecbe521ac9c..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/TopologyAwareTwoNodesMapReduceTest.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.infinispan.configuration.global.GlobalConfigurationBuilder; -import org.infinispan.manager.EmbeddedCacheManager; -import org.infinispan.test.fwk.TestCacheManagerFactory; -import org.testng.annotations.Test; - -/** - * Tests verifying the Map Reduce execution for the Topology Aware nodes. - * - * @author Anna Manukyan - */ -@Test(groups = "functional", testName = "distexec.mapreduce.TopologyAwareTwoNodesMapReduceTest") -public class TopologyAwareTwoNodesMapReduceTest extends SimpleTwoNodesMapReduceTest { - - @Override - protected void createCacheManagers() throws Throwable { - ConfigurationBuilder builder = - getDefaultClusteredCacheConfig(getCacheMode(), false); - - GlobalConfigurationBuilder globalConfigurationBuilder = GlobalConfigurationBuilder.defaultClusteredBuilder(); - globalConfigurationBuilder.transport().machineId("a").rackId("b").siteId("test1"); - - EmbeddedCacheManager cm1 = TestCacheManagerFactory.createClusteredCacheManager(globalConfigurationBuilder, - builder); - cm1.defineConfiguration(cacheName(), builder.build()); - cacheManagers.add(cm1); - - globalConfigurationBuilder = GlobalConfigurationBuilder.defaultClusteredBuilder(); - globalConfigurationBuilder.transport().machineId("b").rackId("b").siteId("test2"); - EmbeddedCacheManager cm2 = TestCacheManagerFactory.createClusteredCacheManager(globalConfigurationBuilder, - builder); - - cm2.defineConfiguration(cacheName(), builder.build()); - cacheManagers.add(cm2); - - waitForClusterToForm(cacheName()); - } - - //Overriding these tests with empty body due to additional cache that is created during these tests. - @Override - public void testEnsureProperCacheState() throws Exception { - - } - - @Override - public void testEnsureProperCacheStateMode() { - - } - -} \ No newline at end of file diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/TwoNodesWithCacheLoaderMapReduceTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/TwoNodesWithCacheLoaderMapReduceTest.java deleted file mode 100644 index 5c0e73d386b3..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/TwoNodesWithCacheLoaderMapReduceTest.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.infinispan.persistence.dummy.DummyInMemoryStoreConfigurationBuilder; -import org.testng.annotations.Test; - -/** - * TwoNodesWithCacheLoaderMapReduceTest tests Map/Reduce functionality using two Infinispan nodes, - * local reduce and also to verify that having values in cacheloader as well does not lead to any - * additional key/value inclusion in map/reduce algorithm - * - * @author Vladimir Blagojevic - * @since 5.2 - */ -@Test(groups = "functional", testName = "distexec.mapreduce.TwoNodesWithCacheLoaderMapReduceTest") -public class TwoNodesWithCacheLoaderMapReduceTest extends BaseWordCountMapReduceTest { - - @Override - protected void createCacheManagers() throws Throwable { - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(getCacheMode(), true); - builder.persistence().addStore(DummyInMemoryStoreConfigurationBuilder.class).storeName(getClass().getSimpleName()).purgeOnStartup(true); - createClusteredCaches(2, cacheName(), builder); - } -} diff --git a/core/src/test/java/org/infinispan/distexec/mapreduce/TwoNodesWithStoreMapReduceTest.java b/core/src/test/java/org/infinispan/distexec/mapreduce/TwoNodesWithStoreMapReduceTest.java deleted file mode 100644 index f876ef64612b..000000000000 --- a/core/src/test/java/org/infinispan/distexec/mapreduce/TwoNodesWithStoreMapReduceTest.java +++ /dev/null @@ -1,102 +0,0 @@ -package org.infinispan.distexec.mapreduce; - -import org.infinispan.Cache; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.infinispan.interceptors.locking.ClusteringDependentLogic; -import org.infinispan.marshall.core.MarshalledEntryImpl; -import org.infinispan.persistence.dummy.DummyInMemoryStoreConfigurationBuilder; -import org.infinispan.persistence.spi.CacheWriter; -import org.infinispan.test.TestingUtil; -import org.testng.annotations.Test; - -/** - * TwoNodesWithStoreMapReduceTest tests Map/Reduce functionality using two Infinispan nodes, - * local reduce and also to verify that having values in cache store only does not lead to any - * additional key/value being missed for map/reduce algorithm - * - * @author Vladimir Blagojevic - * @since 5.2 - */ -@Test(groups = "functional", testName = "distexec.mapreduce.TwoNodesWithStoreMapReduceTest") -public class TwoNodesWithStoreMapReduceTest extends BaseWordCountMapReduceTest { - - @Override - protected void createCacheManagers() throws Throwable { - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(getCacheMode(), true); - builder.persistence().addStore(DummyInMemoryStoreConfigurationBuilder.class).storeName(getClass().getSimpleName()).purgeOnStartup(true); - createClusteredCaches(2, cacheName(), builder); - } - - @Override - @SuppressWarnings({ "rawtypes"}) - public MapReduceTask invokeMapReduce(String keys[], - Mapper mapper, Reducer reducer, boolean useCombiner) - throws Exception { - Cache cache1 = cache(0, cacheName()); - Cache cache2 = cache(1, cacheName()); - - CacheWriter c1 = (CacheWriter) TestingUtil.getCacheLoader(cache1); - CacheWriter c2 = (CacheWriter) TestingUtil.getCacheLoader(cache2); - - //store each entry into each cache store directly - write("1", "Hello world here I am"); - write("1", "Hello world here I am"); - write("2", "Infinispan rules the world"); - write("2", "Infinispan rules the world"); - write("3", "JUDCon is in Boston"); - write("3", "JUDCon is in Boston"); - write("4", "JBoss World is in Boston as well"); - write("4", "JBoss World is in Boston as well"); - write("12", "JBoss Application Server"); - write("12", "JBoss Application Server"); - write("15", "Hello world"); - write("15", "Hello world"); - write("14", "Infinispan community"); - write("14", "Infinispan community"); - - write("111", "Infinispan open source"); - write("111", "Infinispan open source"); - write("112", "Boston is close to Toronto"); - write("112", "Boston is close to Toronto"); - write("113", "Toronto is a capital of Ontario"); - write("113", "Toronto is a capital of Ontario"); - write("114", "JUDCon is cool"); - write("114", "JUDCon is cool"); - write("211", "JBoss World is awesome"); - write("211", "JBoss World is awesome"); - write("212", "JBoss rules"); - write("212", "JBoss rules"); - write("213", "JBoss division of RedHat "); - write("213", "JBoss division of RedHat "); - write("214", "RedHat community"); - write("214", "RedHat community"); - - MapReduceTask task = createMapReduceTask(cache1); - task.mappedWith(mapper).reducedWith(reducer); - if(useCombiner) - task.combinedWith(reducer); - - if(keys != null && keys.length>0){ - task.onKeys(keys); - } - return task; - } - - private void write(String key, Object value) { - Cache cache1 = cache(0, cacheName()); - ClusteringDependentLogic cdl = cache1.getAdvancedCache().getComponentRegistry().getComponent(ClusteringDependentLogic.class); - boolean onCache1 = cdl.localNodeIsPrimaryOwner(key); - CacheWriter cacheWriter; - if (onCache1) { - cacheWriter = (CacheWriter) TestingUtil.getCacheLoader(cache1); - } else { - cacheWriter = (CacheWriter) TestingUtil.getCacheLoader(cache(1, cacheName())); - } - cacheWriter.write(new MarshalledEntryImpl(key, value, null, TestingUtil.marshaller(cache1))); - } - - @Override - public void testInvokeMapReduceOnSubsetOfKeys() throws Exception { - super.testInvokeMapReduceOnSubsetOfKeys(); // TODO: Customise this generated block - } -} diff --git a/core/src/test/java/org/infinispan/remoting/AsynchronousInvocationTest.java b/core/src/test/java/org/infinispan/remoting/AsynchronousInvocationTest.java index 09ae9709b5ff..7e2856078304 100644 --- a/core/src/test/java/org/infinispan/remoting/AsynchronousInvocationTest.java +++ b/core/src/test/java/org/infinispan/remoting/AsynchronousInvocationTest.java @@ -3,7 +3,6 @@ import org.infinispan.Cache; import org.infinispan.commands.CommandsFactory; import org.infinispan.commands.ReplicableCommand; -import org.infinispan.commands.read.ReduceCommand; import org.infinispan.commands.remote.ClusteredGetCommand; import org.infinispan.commands.remote.MultipleRpcCommand; import org.infinispan.commands.remote.SingleRpcCommand; @@ -18,6 +17,7 @@ import org.infinispan.remoting.transport.Transport; import org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher; import org.infinispan.remoting.transport.jgroups.JGroupsTransport; +import org.infinispan.stream.impl.StreamRequestCommand; import org.infinispan.test.AbstractInfinispanTest; import org.infinispan.topology.CacheTopologyControlCommand; import org.infinispan.util.concurrent.BlockingTaskAwareExecutorService; @@ -113,7 +113,7 @@ public void setUp() throws Throwable { ReplicableCommand blockingReplicableCommand = mockReplicableCommand(true); //populate commands - blockingCacheRpcCommand = new ReduceCommand<>("task", null, cacheName, null); + blockingCacheRpcCommand = new StreamRequestCommand<>(cacheName); nonBlockingCacheRpcCommand = new ClusteredGetCommand("key", cacheName, null, false, null, null); blockingNonCacheRpcCommand = new CacheTopologyControlCommand(null, CacheTopologyControlCommand.Type.POLICY_GET_STATUS, null, 0); //the GetKeyValueCommand is not replicated, but I only need a command that returns false in canBlock() diff --git a/core/src/test/java/org/infinispan/stream/stress/DistributedStreamRehashStressTest.java b/core/src/test/java/org/infinispan/stream/stress/DistributedStreamRehashStressTest.java index db5957839c6d..d71683fbdeed 100644 --- a/core/src/test/java/org/infinispan/stream/stress/DistributedStreamRehashStressTest.java +++ b/core/src/test/java/org/infinispan/stream/stress/DistributedStreamRehashStressTest.java @@ -7,7 +7,6 @@ import org.infinispan.configuration.cache.ConfigurationBuilder; import org.infinispan.configuration.global.GlobalConfigurationBuilder; import org.infinispan.container.entries.ImmortalCacheEntry; -import org.infinispan.distexec.mapreduce.Collector; import org.infinispan.distribution.ch.ConsistentHash; import org.infinispan.manager.EmbeddedCacheManager; import org.infinispan.stream.CacheCollectors; @@ -25,7 +24,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.Callable; import java.util.concurrent.Exchanger; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; diff --git a/core/src/test/java/org/infinispan/util/mocks/ControlledCommandFactory.java b/core/src/test/java/org/infinispan/util/mocks/ControlledCommandFactory.java index 63adb80fe4b2..9ac2c4138967 100644 --- a/core/src/test/java/org/infinispan/util/mocks/ControlledCommandFactory.java +++ b/core/src/test/java/org/infinispan/util/mocks/ControlledCommandFactory.java @@ -24,8 +24,6 @@ import org.infinispan.commands.read.GetCacheEntryCommand; import org.infinispan.commands.read.GetKeyValueCommand; import org.infinispan.commands.read.KeySetCommand; -import org.infinispan.commands.read.MapCombineCommand; -import org.infinispan.commands.read.ReduceCommand; import org.infinispan.commands.read.SizeCommand; import org.infinispan.commands.remote.ClusteredGetAllCommand; import org.infinispan.commands.remote.ClusteredGetCommand; @@ -53,8 +51,6 @@ import org.infinispan.commands.write.WriteCommand; import org.infinispan.commons.api.functional.EntryView; import org.infinispan.context.Flag; -import org.infinispan.distexec.mapreduce.Mapper; -import org.infinispan.distexec.mapreduce.Reducer; import org.infinispan.factories.ComponentRegistry; import org.infinispan.functional.impl.Params; import org.infinispan.manager.EmbeddedCacheManager; @@ -319,16 +315,6 @@ public DistributedExecuteCommand buildDistributedExecuteCommand(Callable< return actual.buildDistributedExecuteCommand(callable, sender, keys); } - @Override - public MapCombineCommand buildMapCombineCommand(String taskId, Mapper m, Reducer r, Collection keys) { - return actual.buildMapCombineCommand(taskId, m, r, keys); - } - - @Override - public ReduceCommand buildReduceCommand(String taskId, String destinationCache, Reducer r, Collection keys) { - return actual.buildReduceCommand(taskId, destinationCache, r, keys); - } - @Override public GetInDoubtTxInfoCommand buildGetInDoubtTxInfoCommand() { return actual.buildGetInDoubtTxInfoCommand(); diff --git a/demos/distexec/src/main/java/org/infinispan/demo/mapreduce/WordCountCollator.java b/demos/distexec/src/main/java/org/infinispan/demo/mapreduce/WordCountCollator.java deleted file mode 100644 index a119589b8568..000000000000 --- a/demos/distexec/src/main/java/org/infinispan/demo/mapreduce/WordCountCollator.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.infinispan.demo.mapreduce; - -import org.infinispan.distexec.mapreduce.Collator; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * Collates reduced results by grouping them into the top K most frequent words. - * - * @author Vladimir Blagojevic - */ -public class WordCountCollator implements Collator>> { - - private final int kthFrequentWord; - - public WordCountCollator() { - this.kthFrequentWord = 10; - } - - public WordCountCollator(int kthFrequentWord) { - if (kthFrequentWord < 0) - throw new IllegalArgumentException("kth FrequentWord can not be less than 0"); - this.kthFrequentWord = kthFrequentWord; - } - - @Override - public List> collate(Map reducedResults) { - // Set Thread Name - Thread.currentThread().setName(String.format("CollatorThread-%d", Thread.currentThread().getId())); - - Set> entrySet = reducedResults.entrySet(); - ArrayList> l = new ArrayList>(entrySet); - Collections.sort(l, new Comparator>() { - - @Override - public int compare(Map.Entry o1, Map.Entry o2) { - return o1.getValue() < o2.getValue() ? 1 : o1.getValue() > o2.getValue() ? -1 : 0; - } - }); - - List> results = new LinkedList>(); - for (int i=0; i cache = startCache(); - - // Step 2: load up data file - if (textFile != null) loadData(cache); - - // Step 3: if slave, wait. Else, start Map/Reduce task. - try { - if (isMaster) { - long start = System.currentTimeMillis(); - MapReduceTask mapReduceTask = new MapReduceTask(cache); - - List> topList = - mapReduceTask - .mappedWith(new WordCountMapper()) - .reducedWith(new WordCountReducer()) - .execute(new WordCountCollator(numPopularWords)); - - - System.out.printf(" ---- RESULTS: Top %s words in %s ---- %n%n", numPopularWords, textFile); - int z = 0; - for (Entry e : topList) - System.out.printf(" %s) %s [ %,d occurences ]%n", ++z, e.getKey(), e.getValue()); - System.out.printf("%nCompleted in %s%n%n", Util.prettyPrintTime(System.currentTimeMillis() - start)); - } else { - System.out.println("Slave node waiting for Map/Reduce tasks. Ctrl-C to exit."); - LockSupport.park(); - } - } finally { - cache.getCacheManager().stop(); - } - } - - private void loadData(Cache cache) throws IOException { - FileReader in = new FileReader(textFile); - try(BufferedReader bufferedReader = new BufferedReader(in)) { - //chunk and insert into cache - int chunkSize = 10; // 10K - int chunkId = 0; - - CharBuffer cbuf = CharBuffer.allocate(1024 * chunkSize); - while (bufferedReader.read(cbuf) >= 0) { - Buffer buffer = cbuf.flip(); - String textChunk = buffer.toString(); - cache.put(textFile + (chunkId++), textChunk); - cbuf.clear(); - if (chunkId % 100 == 0) System.out.printf(" Inserted %s chunks from %s into grid%n", chunkId, textFile); - } - } finally { - Util.close(in); - } - } - - @Override - protected SimpleJSAP buildCommandLineOptions() throws JSAPException { - return new SimpleJSAP( - "WordCountDemo", - "Count words in Infinispan cache usin MapReduceTask ", - new Parameter[]{ - new FlaggedOption("configFile", JSAP.STRING_PARSER, "configs/config-samples/distributed-udp.xml", - JSAP.NOT_REQUIRED, 'c', "configFile", - "Infinispan transport config file"), - new FlaggedOption("nodeType", JSAP.STRING_PARSER, "slave", JSAP.REQUIRED, - 't', "nodeType", "Node type as either master or slave"), - new FlaggedOption("textFile", JSAP.STRING_PARSER, null, JSAP.NOT_REQUIRED, - 'f', "textFile", "Input text file to distribute onto grid"), - new FlaggedOption("mostPopularWords", JSAP.INTEGER_PARSER, "15", - JSAP.NOT_REQUIRED, 'n', "mostPopularWords", "Number of most popular words to find")}); - } -} diff --git a/demos/distexec/src/main/java/org/infinispan/demo/mapreduce/WordCountMapper.java b/demos/distexec/src/main/java/org/infinispan/demo/mapreduce/WordCountMapper.java deleted file mode 100644 index fbb529caec9a..000000000000 --- a/demos/distexec/src/main/java/org/infinispan/demo/mapreduce/WordCountMapper.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.infinispan.demo.mapreduce; - -import java.util.Calendar; - -import org.infinispan.distexec.mapreduce.Collector; -import org.infinispan.distexec.mapreduce.Mapper; - -public class WordCountMapper implements Mapper { - private static final long serialVersionUID = -5943370243108735560L; - private static int chunks = 0, words = 0; - - @Override - public void map(String key, String value, Collector c) { - // Set Thread Name - Thread.currentThread().setName(String.format("MapperThread-%d", Thread.currentThread().getId())); - - chunks++; - /* - * Split on punctuation or whitespace, except for ' and - to catch contractions and hyphenated - * words - */ - for (String word : value.split("[\\p{Punct}\\s&&[^'-]]+")) { - if (word != null) { - String w = word.trim(); - if (w.length() > 0) { - c.emit(word.toLowerCase(), 1); - words++; - } - } - } - - if (chunks % 1000 == 0) - System.out.printf("%tT Analyzed %s words in %s lines%n", Calendar.getInstance(), words, chunks); - } -} \ No newline at end of file diff --git a/demos/distexec/src/main/java/org/infinispan/demo/mapreduce/WordCountMapperEmitPerValue.java b/demos/distexec/src/main/java/org/infinispan/demo/mapreduce/WordCountMapperEmitPerValue.java deleted file mode 100644 index 7bf7e45437ac..000000000000 --- a/demos/distexec/src/main/java/org/infinispan/demo/mapreduce/WordCountMapperEmitPerValue.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.infinispan.demo.mapreduce; - -import java.util.Calendar; -import java.util.HashMap; -import java.util.Map; - -import org.infinispan.distexec.mapreduce.Collector; -import org.infinispan.distexec.mapreduce.Mapper; - -public class WordCountMapperEmitPerValue implements Mapper { - private static final long serialVersionUID = -5943370243108735560L; - private int values = 0, words = 0; - - public WordCountMapperEmitPerValue() { - super(); - // Reset counters - values = 0; - words = 0; - } - - @Override - public void map(String key, String value, Collector c) { - HashMap results = new HashMap(); - values++; - - // Set Thread Name - Thread.currentThread().setName(String.format("MapperThread-%d", Thread.currentThread().getId())); - - /* - * Split on punctuation or whitespace, except for ' and - to catch contractions and hyphenated - * words - */ - for (String word : value.split("[\\p{Punct}\\s&&[^'-]]+")) { - if (word.length() > 0) { - if (results.containsKey(word)) { - results.put(word, results.get(word) + 1); - } else { - results.put(word, 1); - words++; - } - } - } - - for (Map.Entry entry : results.entrySet()) { - c.emit(entry.getKey().toLowerCase(), entry.getValue()); - } - - if (values % 5000 == 0) { - System.out.printf("%tT %s Analyzed %s words in %s lines%n", Calendar.getInstance(), Thread.currentThread() - .toString(), words, values); - } - } -} diff --git a/demos/distexec/src/main/java/org/infinispan/demo/mapreduce/WordCountReducer.java b/demos/distexec/src/main/java/org/infinispan/demo/mapreduce/WordCountReducer.java deleted file mode 100644 index 6aa1fb0e7b49..000000000000 --- a/demos/distexec/src/main/java/org/infinispan/demo/mapreduce/WordCountReducer.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.infinispan.demo.mapreduce; - -import org.infinispan.distexec.mapreduce.Reducer; - -import java.util.Iterator; - -public class WordCountReducer implements Reducer { - - private static final long serialVersionUID = 1901016598354633256L; - - @Override - public Integer reduce(String key, Iterator iter) { - // Set Thread Name - Thread.currentThread().setName(String.format("ReducerThread-%d", Thread.currentThread().getId())); - - int sum = 0; - while (iter.hasNext()) { - Integer i = iter.next(); - sum += i; - } - return sum; - } -} diff --git a/documentation/src/main/asciidoc/upgrading/upgrading.adoc b/documentation/src/main/asciidoc/upgrading/upgrading.adoc index 08740382e69d..15a82a86c9d8 100644 --- a/documentation/src/main/asciidoc/upgrading/upgrading.adoc +++ b/documentation/src/main/asciidoc/upgrading/upgrading.adoc @@ -13,6 +13,10 @@ The entry retriever feature has been removed. Please update to use the new Stre The `org.infinispan.filter.CacheFilters` class can be used to convert `KeyValueFilter` and `Converter` instances into proper Stream operations that are able to be marshalled. +=== Map / Reduce is now removed +Map reduce has been removed in favor of the new Streams feature which should provide more features and performance. +There are no bridge classes to convert to the new streams and all references must be rewritten. + === Function classes have moved packages The class `SerializableSupplier` has moved from the `org.infinispan.stream` package to the `org.infinispan.util.function` package. diff --git a/documentation/src/main/asciidoc/user_guide/chapter-6-Querying_Infinispan.adoc b/documentation/src/main/asciidoc/user_guide/chapter-6-Querying_Infinispan.adoc index 494c5d54c678..2d435e734d2f 100644 --- a/documentation/src/main/asciidoc/user_guide/chapter-6-Querying_Infinispan.adoc +++ b/documentation/src/main/asciidoc/user_guide/chapter-6-Querying_Infinispan.adoc @@ -330,8 +330,7 @@ WARNING: This is a new API undergoing refinements and changes that might break c Starting with 6.0 Infinispan provides its own query DSL, independent of Lucene and Hibernate Search. Decoupling the query API from the underlying query and indexing mechanism makes it possible to introduce new alternative engines in the future, besides Lucene, and still being able to use the same uniform query API. -This opens up the possibility of introducing a future index-less searching engine based on Map/Reduce that will be -accessible exclusively via the new query API. The current implementation of indexing and searching is still based on +The current implementation of indexing and searching is still based on Hibernate Search and Lucene so all indexing related aspects presented in this chapter still apply. The new API simplifies the writing of queries by not exposing the user to the low level details of constructing Lucene @@ -441,10 +440,6 @@ qb2.having("description").like("%clustering%"); // will throw java.lang.Illega Query q2 = qb2.build(); ---- -NOTE: All fields participating in the query need to be indexed by Hibernate Search for querying to work in embedded -mode. We plan to add index-less searching with Map/Reduce in future versions -(see link:$$https://issues.jboss.org/browse/ISPN-3717$$[ISPN-3717]). - ==== Filtering based on attributes of embedded entities The `having` method also accepts dot separated attribute paths for referring to _embedded entity_ attributes, so the following diff --git a/documentation/src/main/asciidoc/user_guide/chapter-63-Streams.adoc b/documentation/src/main/asciidoc/user_guide/chapter-63-Streams.adoc index 79417b556acb..407080c0c056 100644 --- a/documentation/src/main/asciidoc/user_guide/chapter-63-Streams.adoc +++ b/documentation/src/main/asciidoc/user_guide/chapter-63-Streams.adoc @@ -321,8 +321,7 @@ Then finally all of those results have a distinct operation performed upon them. The rest of the intermediate operations are fully distributed as one would expect. === Examples -Using the existing Map/Reduce example link:user_guide.html#examples_2[example], -it has been adapted for Streams. Word count is a classic, if overused, example +Word count is a classic, if overused, example of map/reduce paradigm. Assume we have a mapping of key -> sentence stored on Infinispan nodes. Key is a String, each sentence is also a String, and we have to count occurrence of all words in all sentences available. The implementation @@ -375,8 +374,9 @@ However what if we want to find the most frequent word in the example? If you t to think about this case you will realize you need to have all words counted and available locally first. Thus we actually have a few options. -We can use something akin to the old Collator from Map/Reduce by slightly tweaking -our collector. Some redundant lines have been removed from the previous example. +We could use a finisher on the collector, which is invoked on the user thread +after all the results have been collected. +Some redundant lines have been removed from the previous example. [source,java] ---- diff --git a/documentation/src/main/asciidoc/user_guide/chapter-66-Map_Reduce_framework.adoc b/documentation/src/main/asciidoc/user_guide/chapter-66-Map_Reduce_framework.adoc deleted file mode 100644 index 9410d96aa66e..000000000000 --- a/documentation/src/main/asciidoc/user_guide/chapter-66-Map_Reduce_framework.adoc +++ /dev/null @@ -1,240 +0,0 @@ -== Map/Reduce - -MapReduce is a programming model allowing transparent distributed processing of very large data sets over data grids. The name MapReduce comes from an idea of using two distinct computational phases of map and reduce. In the map phase, master node that initiates a task takes the task input, divides it and sends tasks for map phase execution on the grid. Each node in turns executes a map function on its input returning intermediate results back to master node. Master node task collects all intermediate results from map phase combines them by intermediate result keys and sends intermediate keys/values for reduction on the grid. Finally master tasks node receives all results from reduction phases and returns the final result to invoker of the MapReduce task. - -CAUTION: The Map/Reduce framework has been deprecated and will be removed with -Infinispan 9. Please check out the link:user_guide.html#_streams[Streams] -section for its replacement. - -=== API -Infinispan's own MapReduce model is an adaptation of Google's original link:$$http://research.google.com/archive/mapreduce.html$$[MapReduce] . There are four main components in each map reduce task: Mapper, Reducer, Collator and MapReduceTask. - -Implementation of a Mapper class is a component of MapReduceTask invoked once for each input entry K,V. Every Mapper instance is migrated to an Infinispan node, given a cache entry K,V input pair which it transforms into an intermediate key/value pair emitted into Infinispan provided Collector instance. Intermediate results are further reduced using a Reducer. - -[source,java] ----- - -public interface Mapper extends Serializable { - - /** - * Invoked once for each input cache entry KIn,VOut pair. - */ - void map(KIn key, VIn value, Collector collector); -} - ----- - -The Reducer, as its name implies, reduces a list of intermediate results from map phase of MapReduceTask. Infinispan distributed execution environment creates one instance of Reducer per execution node. - -[source,java] ----- -public interface Reducer extends Serializable { - - /** - * Combines/reduces all intermediate values for a particular intermediate key to a single value. - *

- * - */ - VOut reduce(KOut reducedKey, Iterator iter); - -} ----- - -Collator coordinates results from Reducers executed on Infinispan cluster and assembles a final result returned to an invoker of MapReduceTask. Collator is applied to final `Map` result of MapReduceTask. - -[source,java] ----- -public interface Reducer extends Serializable { - - /** - * Combines/reduces all intermediate values for a particular intermediate key to a single value. - *

- * - */ - VOut reduce(KOut reducedKey, Iterator iter); - -} ----- - -Finally, link:$$https://github.com/infinispan/infinispan/blob/master/core/src/main/java/org/infinispan/distexec/mapreduce/MapReduceTask.java$$[MapReduceTask] is a distributed task unifying Mapper, Reducer and Collator into a cohesive large scale computation to be transparently parallelized across Infinispan cluster nodes. Users of MapReduceTask need to provide a cache whose data is used as input for this task. Infinispan execution environment will instantiate and migrate instances of provided mappers and reducers seamlessly across Infinispan nodes. Unless otherwise specified using onKeys method input keys filter all available key value pairs of a specified cache will be used as input data for this task. - -==== Task timeout - -NOTE: The timeout per Map/Reduce task is only available for Infinispan 5.3 and higher. - -It is possible to set a timeout value for each Map/Reduce tasks. However, if no timeout is specified, it uses the replication timeout as a default timeout (the same behavior as the previous Infinispan versions). You can set the timeout in your task by doing the following: - -[source,java] ----- -MapReduceTask task = new MapReduceTask(cache); -task.timeout(1, TimeUnit.MINUTES); ----- - -Also, it is possible to know which is the current timeout value for the task: - -[source,java] ----- -System.out.println("Map/Reduce task timeout is " + task.timeout(TimeUnit.MILLISECONDS) + " millseconds");  ----- - -For more information about this, please check the java doc in link:http://docs.jboss.org/infinispan/{infinispanversion}/apidocs/org/infinispan/distexec/mapreduce/MapReduceTask.html[Map Reduce Task API Documentation] - -=== Mapper and CDI - -Although Mapper gets invoked with an appropriate input key/value pairs on an executing node, Infinispan also provides CDI injection of an input Cache in case users might need some additional data from input cache in order to complete map transformation. Upon arrival of user's Mapper to an Infinispan executing node, Infinispan CDI mechanism will provide appropriate cache reference and inject it to executing Mapper. All one has to do is to declare a Cache field in Mapper and annotate it with @org.infinispan.cdi.Input annotation along with the mandatory @Inject annotation. - -[source,java] ----- -public class WordCountCacheInjectedMapper implements Mapper { - - @Inject - @Input - private Cache cache; - - @Override - public void map(String key, String value, Collector collector) { - - //use injected cache if needed - StringTokenizer tokens = new StringTokenizer(value); - while (tokens.hasMoreElements()) { - String s = (String) tokens.nextElement(); - collector.emit(s, 1); - } - } -} ----- - -=== MapReduceTask distributed execution -As our MapReduce implementation grew out of the proof of concept phase (and especially after our users had already production tested it), we needed to remove the most prominent impediment to an industrial grade MapReduce solution that we strive for: distributing reduce phase execution. - -* Reduce phase: prior to the Infinispan 5.2 release was done on a single Infinispan master task node. Therefore, the size of map reduce problems we could support (data size wise) was effectively shrunk to a working memory of a single Infinispan node. Starting with the Infinispan 5.2 release, we have removed this limitation, and reduce phase execution is distributed across the cluster as well. Of course, users still have an option to use MapReduceTask the old way, and we even recommend that particular approach for smaller sized input tasks. We have achieved distribution of reduce phase by relying on Infinispan's consistent hashing and DeltaAware cache insertion. Here is how we distributed reduce phase execution: - -* Map phase: MapReduceTask, as it currently does, will hash task input keys and group them by execution node N they are hashed to. After key node mapping, MapReduceTask sends map function and input keys to each node N. Map function is invoked using given keys and locally loaded corresponding values. - -image::images/intermediatephase.png[] - -* Results are collected with an Infinispan supplied Collector, and combine phase is initiated. A Combiner, if specified, takes KOut keys and immediately invokes reduce phase on keys. The result of mapping phase executed on each node is KOut/VOut map. There will be one resulting map per execution node N per launched MapReduceTask. - -* Intermediate KOut/VOut migration phase -** In order to proceed with reduce phase, all intermediate keys and values need to be grouped by intermediate KOut keys. More specifically, as map phases around the cluster can produce identical intermediate keys, all those identical intermediate keys and their values need to be grouped before reduce is executed on any particular intermediate key. -** Therefore at the end of combine phase, instead of returning map with intermediate keys and values to the master task node, we instead hash each intermediate key KOut and migrate it with its VOut values to Infinispan node where keys KOut are hashed to. We achieve this using a temporary DIST cache and underlying consistent hashing mechanism. Using DeltaAware cache insertion we effectively collect all VOut values under each KOut for all executed map functions across the cluster - - -image::images/mapphase.png[] - -* At this point, map and combine phase have finished its execution; list of KOut keys is returned to a master node and its initiating MapReduceTask. We do not return VOut values as we do not need them at master task node. MapReduceTask is ready to start with reduce phase. - -Reduce phase -* Reduce phase is easy to accomplish now as Infinispan's consistent hashing already finished all the hard lifting for us. To complete reduce phase, MapReduceTask groups KOut keys by execution node N they are hashed to. For each node N and its grouped input KOut keys, MapReduceTask sends a reduce command to a node N where KOut keys are hashed. Once reduce command arrives on target execution node, it looks up temporary cache belonging to MapReduce task - and for each KOut key, grabs a list of VOut values, wraps it with an Iterator and invokes reduce on it._ - - -image::images/reducephase.png[] - -A result of each reduce is a map where each key is KOut and value is VOut. Each Infinispan execution node N returns one map with KOut/VOut result values. As all initiated reduce commands return to a calling node, MapReduceTask simply combines all resulting maps into map M and returns M as a result of MapReduceTask. - -Distributed reduce phase is turned on by using a MapReduceTask link:http://docs.jboss.org/infinispan/{infinispanversion}/apidocs/org/infinispan/distexec/mapreduce/MapReduceTask.html#MapReduceTask(org.infinispan.Cache, boolean)[constructor] specifying cache to use as input data for the task and boolean parameter distributeReducePhase set to true. Map/Reduce API link:http://docs.jboss.org/infinispan/{infinispanversion}/apidocs/org/infinispan/distexec/mapreduce/package-summary.html[javadoc] and demos are included in distribution. - -=== Examples -Word count is a classic, if not overused, example of map/reduce paradigm. Assume we have a mapping of key --> sentence stored on Infinispan nodes. Key is a String, each sentence is also a String, and we have to count occurrence of all words in all sentences available. The implementation of such a distributed task could be defined as follows: - -[source,java] ----- -public class WordCountExample { - - /** - * In this example replace c1 and c2 with - * real Cache references - * - * @param args - */ - public static void main(String[] args) { - Cache c1 = null; - Cache c2 = null; - - c1.put("1", "Hello world here I am"); - c2.put("2", "Infinispan rules the world"); - c1.put("3", "JUDCon is in Boston"); - c2.put("4", "JBoss World is in Boston as well"); - c1.put("12","JBoss Application Server"); - c2.put("15", "Hello world"); - c1.put("14", "Infinispan community"); - c2.put("15", "Hello world"); - - c1.put("111", "Infinispan open source"); - c2.put("112", "Boston is close to Toronto"); - c1.put("113", "Toronto is a capital of Ontario"); - c2.put("114", "JUDCon is cool"); - c1.put("211", "JBoss World is awesome"); - c2.put("212", "JBoss rules"); - c1.put("213", "JBoss division of RedHat "); - c2.put("214", "RedHat community"); - - MapReduceTask t = - new MapReduceTask(c1); - t.mappedWith(new WordCountMapper()) - .reducedWith(new WordCountReducer()); - Map wordCountMap = t.execute(); - } - - static class WordCountMapper implements Mapper { - /** The serialVersionUID */ - private static final long serialVersionUID = -5943370243108735560L; - - @Override - public void map(String key, String value, Collector c) { - StringTokenizer tokens = new StringTokenizer(value); - while (tokens.hasMoreElements()) { - String s = (String) tokens.nextElement(); - c.emit(s, 1); - } - } - } - - static class WordCountReducer implements Reducer { - /** The serialVersionUID */ - private static final long serialVersionUID = 1901016598354633256L; - - @Override - public Integer reduce(String key, Iterator iter) { - int sum = 0; - while (iter.hasNext()) { - Integer i = (Integer) iter.next(); - sum += i; - } - return sum; - } - } -} - ----- - -As we have seen it is relatively easy to specify map reduce task counting number of occurrences for each word in all sentences. Best of all result is returned to task invoker in the form of `Map` rather than being written to a stream. - -What if we need to find the most frequent word in our word count example? All we have to do is to define a Collator that will transform the result of MapReduceTask `Map` into a String which in turn is returned to a task invoker. We can think of Collator as transformation function applied to a final result of MapReduceTask. - -[source,java] ----- -MapReduceTask t = new MapReduceTask(cache); -t.mappedWith(new WordCountMapper()).reducedWith(new WordCountReducer()); -String mostFrequentWord = t.execute( - new Collator() { - - @Override - public String collate(Map reducedResults) { - String mostFrequent = ""; - int maxCount = 0; - for (Entry e : reducedResults.entrySet()) { - Integer count = e.getValue(); - if(count > maxCount) { - maxCount = count; - mostFrequent = e.getKey(); - } - } - return mostFrequent; - } - - }); -System.out.println("The most frequent word is " + mostFrequentWord); - ----- - diff --git a/documentation/src/main/asciidoc/user_guide/chapter-75-Security.adoc b/documentation/src/main/asciidoc/user_guide/chapter-75-Security.adoc index 696d9d17d4df..ac5ec0447b24 100644 --- a/documentation/src/main/asciidoc/user_guide/chapter-75-Security.adoc +++ b/documentation/src/main/asciidoc/user_guide/chapter-75-Security.adoc @@ -31,7 +31,7 @@ Access to a cache manager or a cache is controlled by using a list of required p ===== Cache permissions * READ (get, contains): whether entries can be retrieved from the cache * WRITE (put, putIfAbsent, replace, remove, evict): whether data can be written/replaced/removed/evicted from the cache -* EXEC (distexec, mapreduce): whether code execution can be run against the cache +* EXEC (distexec, streams): whether code execution can be run against the cache * LISTEN (addListener): whether listeners can be registered against a cache * BULK_READ (keySet, values, entrySet, query): whether bulk retrieve operations can be executed * BULK_WRITE (clear, putAll): whether bulk write operations can be executed diff --git a/documentation/src/main/asciidoc/user_guide/chapter-77-Simple_Cache.adoc b/documentation/src/main/asciidoc/user_guide/chapter-77-Simple_Cache.adoc index eb975c5b6f9b..4371a873bc0e 100644 --- a/documentation/src/main/asciidoc/user_guide/chapter-77-Simple_Cache.adoc +++ b/documentation/src/main/asciidoc/user_guide/chapter-77-Simple_Cache.adoc @@ -17,7 +17,6 @@ So, which features are stripped away? From the configuration perspective, simple From the API perspective these features throw an exception: * adding custom interceptors -* Map Reduce Framework * Distributed Executors Framework So, what's left? @@ -50,4 +49,4 @@ Cache cache = cm.getCache("mySimpleCache"); ---- Simple cache checks against features it does not support, if you configure it to use e.g. transactions, -configuration validation will throw an exception. \ No newline at end of file +configuration validation will throw an exception. diff --git a/documentation/src/main/asciidoc/user_guide/chapter-78-Scripting.adoc b/documentation/src/main/asciidoc/user_guide/chapter-78-Scripting.adoc index 9711115a439e..9d8ab17a9ac4 100644 --- a/documentation/src/main/asciidoc/user_guide/chapter-78-Scripting.adoc +++ b/documentation/src/main/asciidoc/user_guide/chapter-78-Scripting.adoc @@ -30,16 +30,9 @@ The following metadata property keys are available * mode: defines the mode of execution of a script. Can be one of the following values: ** local: the script will be executed only by the node handling the request. The script itself however can invoke clustered operations ** distributed: runs the script using the Distributed Executor Service -** mapper: runs the script as the mapper of map/reduce job. This property also requires that the user specify a +reducer+ property and optional +collator+ and +combiner+ properties. This mode is deprecated and will be removed in Infinispan 9. Use local mode and the distributed streams API instead. -** reducer: this type of script can't be executed directly, but must be specified as a +reducer+ property on a script of type +mapper+. This mode is deprecated and will be removed in Infinispan 9. Use local mode and the distributed streams API instead. -** collator: this type of script can't be executed directly, but must be specified as a +collator+ property on a script of type +mapper+. This mode is deprecated and will be removed in Infinispan 9. Use local mode and the distributed streams API instead. -** combiner: this type of script can't be executed directly, but must be specified as a +combiner+ property on a script of type +mapper+. This mode is deprecated and will be removed in Infinispan 9. Use local mode and the distributed streams API instead. * language: defines the script engine that will be used to execute the script, e.g. Javascript * extension: an alternative method of specifying the script engine that will be used to execute the script, e.g. js * role: a specific role which is required to execute the script -* reducer: this property is only valid for +mapper+ scripts, and specifies the name of another script that will be used for the reduce phase of a map/reduce job. This property will be removed in Infinispan 9. -* collator: this property is only valid for +mapper+ scripts, and specifies the name of another script that will be used for the collation phase of a map/reduce job. This property will be removed in Infinispan 9. -* combiner: this property is only valid for +mapper+ scripts, and specifies the name of another script that will be used for the combine phase of a map/reduce job. This property will be removed in Infinispan 9. * parameters: an array of valid parameter names for this script. Invocations which specify parameter names not included in this list will cause an exception. * datatype: optional property providing information, in the form of Media Types (also known as MIME) about the type of the data stored in the diff --git a/documentation/src/main/asciidoc/user_guide/user_guide.adoc b/documentation/src/main/asciidoc/user_guide/user_guide.adoc index f6db18ab8ecf..3d3975c60d25 100644 --- a/documentation/src/main/asciidoc/user_guide/user_guide.adoc +++ b/documentation/src/main/asciidoc/user_guide/user_guide.adoc @@ -34,7 +34,6 @@ include::chapter-61-Marshalling.adoc[] include::chapter-14-Infinispan_Server_Modules.adoc[] include::chapter-6-Querying_Infinispan.adoc[] include::chapter-13-Infinispan_as_a_storage_for_Lucene_indexes.adoc[] -include::chapter-66-Map_Reduce_framework.adoc[] include::chapter-63-Streams.adoc[] include::chapter-41-Infinispan_Distributed_Execution_Framework.adoc[] include::chapter-15-Management_Tooling.adoc[] diff --git a/integrationtests/all-embedded-it/src/test/java/org/infinispan/all/embedded/EmbeddedAllTest.java b/integrationtests/all-embedded-it/src/test/java/org/infinispan/all/embedded/EmbeddedAllTest.java index a9659f9a9205..5b5d7c120267 100644 --- a/integrationtests/all-embedded-it/src/test/java/org/infinispan/all/embedded/EmbeddedAllTest.java +++ b/integrationtests/all-embedded-it/src/test/java/org/infinispan/all/embedded/EmbeddedAllTest.java @@ -20,10 +20,6 @@ import org.infinispan.configuration.global.GlobalConfiguration; import org.infinispan.configuration.global.GlobalConfigurationBuilder; import org.infinispan.distexec.DefaultExecutorService; -import org.infinispan.distexec.mapreduce.Collector; -import org.infinispan.distexec.mapreduce.MapReduceTask; -import org.infinispan.distexec.mapreduce.Mapper; -import org.infinispan.distexec.mapreduce.Reducer; import org.infinispan.lifecycle.ComponentStatus; import org.infinispan.manager.DefaultCacheManager; import org.infinispan.manager.EmbeddedCacheManager; @@ -195,37 +191,6 @@ public String call() throws Exception { } } - @Test - public void testEmbeddedMapReduce() throws Exception { - Cache cache = manager.getCache(); - cache.put("k1", "v1"); - cache.put("k2", "v1"); - MapReduceTask task = new MapReduceTask(cache); - task.mappedWith(new TestMapper()).reducedWith(new TestReducer()); - Map result = task.execute(); - assertEquals(1, result.size()); - assertEquals(2, result.get("v1").intValue()); - } - - static class TestMapper implements Mapper { - public void map(String key, String value, Collector collector) { - collector.emit(value, 1); - } - } - - static class TestReducer implements Reducer { - - public Integer reduce(String reducedKey, Iterator iter) { - int count = 0; - for(; iter.hasNext(); iter.next()) { - count++; - } - return count; - } - - } - - private void testDataSurvived(Cache cache) { String key1 = "key1_" + cache.getName(); String key2 = "key2_" + cache.getName(); diff --git a/integrationtests/osgi/src/test/java/org/infinispan/it/osgi/distexec/mapreduce/BaseWordCountMapReduceTest.java b/integrationtests/osgi/src/test/java/org/infinispan/it/osgi/distexec/mapreduce/BaseWordCountMapReduceTest.java deleted file mode 100644 index b19c49447368..000000000000 --- a/integrationtests/osgi/src/test/java/org/infinispan/it/osgi/distexec/mapreduce/BaseWordCountMapReduceTest.java +++ /dev/null @@ -1,100 +0,0 @@ -package org.infinispan.it.osgi.distexec.mapreduce; - -import static org.infinispan.it.osgi.util.IspnKarafOptions.perSuiteOptions; -import static org.ops4j.pax.exam.CoreOptions.options; - -import org.infinispan.commons.CacheException; -import org.junit.BeforeClass; -import org.junit.Test; -import org.ops4j.pax.exam.Configuration; -import org.ops4j.pax.exam.Option; - -/** - * @author mgencur - */ -public abstract class BaseWordCountMapReduceTest extends org.infinispan.distexec.mapreduce.BaseWordCountMapReduceTest { - @Configuration - public Option[] config() throws Exception { - return options(perSuiteOptions()); - } - - @BeforeClass - public static void setUpBeforeClass() throws Throwable { - BaseWordCountMapReduceTest.specifyWordCounts(); - } - - @Test - public void testInvokeMapReduceOnAllKeys() throws Exception { - super.testInvokeMapReduceOnAllKeys(); - } - - @Test - public void testInvokeMapReduceOnAllKeysWithResultCache() throws Exception { - super.testInvokeMapReduceOnAllKeysWithResultCache(); - } - - @Test - public void testInvokeMapReduceOnEmptyKeys() throws Exception { - super.testInvokeMapReduceOnEmptyKeys(); - } - - @Test - public void testInvokeMapReduceOnAllKeysWithCombiner() throws Exception { - super.testInvokeMapReduceOnAllKeysWithCombiner(); - } - - @Test - public void testCombinerDoesNotChangeResult() throws Exception { - super.testCombinerDoesNotChangeResult(); - } - - @Test - public void testMapperReducerIsolation() throws Exception{ - super.testMapperReducerIsolation(); - } - - @Test - public void testInvokeMapReduceOnAllKeysAsync() throws Exception { - super.testInvokeMapReduceOnAllKeysAsync(); - } - - @Test - public void testInvokeMapReduceOnSubsetOfKeys() throws Exception { - super.testInvokeMapReduceOnSubsetOfKeys(); - } - - @Test - public void testInvokeMapReduceOnSubsetOfKeysWithResultCache() throws Exception { - super.testInvokeMapReduceOnSubsetOfKeysWithResultCache(); - } - - @Test - public void testInvokeMapReduceOnSubsetOfKeysAsync() throws Exception { - super.testInvokeMapReduceOnSubsetOfKeysAsync(); - } - - @Test - public void testInvokeMapReduceOnAllKeysWithCollator() throws Exception { - super.testInvokeMapReduceOnAllKeysWithCollator(); - } - - @Test - public void testInvokeMapReduceOnSubsetOfKeysWithCollator() throws Exception { - super.testInvokeMapReduceOnSubsetOfKeysWithCollator(); - } - - @Test - public void testInvokeMapReduceOnAllKeysWithCollatorAsync() throws Exception { - super.testInvokeMapReduceOnAllKeysWithCollatorAsync(); - } - - @Test - public void testInvokeMapReduceOnSubsetOfKeysWithCollatorAsync() throws Exception { - super.testInvokeMapReduceOnSubsetOfKeysWithCollatorAsync(); - } - - @Test - public void testCombinerForDistributedReductionWithException() throws Exception { - super.testCombinerForDistributedReductionWithException(); - } -} diff --git a/integrationtests/osgi/src/test/java/org/infinispan/it/osgi/distexec/mapreduce/DistributedTwoNodesMapReduceTest.java b/integrationtests/osgi/src/test/java/org/infinispan/it/osgi/distexec/mapreduce/DistributedTwoNodesMapReduceTest.java deleted file mode 100644 index 56233ea4d9f7..000000000000 --- a/integrationtests/osgi/src/test/java/org/infinispan/it/osgi/distexec/mapreduce/DistributedTwoNodesMapReduceTest.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.infinispan.it.osgi.distexec.mapreduce; - -import org.infinispan.Cache; -import org.infinispan.configuration.cache.ConfigurationBuilder; -import org.infinispan.distexec.mapreduce.MapReduceTask; -import org.infinispan.test.TestingUtil; -import org.infinispan.test.fwk.TestResourceTracker; -import org.junit.After; -import org.junit.Before; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.ops4j.pax.exam.junit.PaxExam; -import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy; -import org.ops4j.pax.exam.spi.reactors.PerSuite; - -/** - * @author mgencur - */ -@RunWith(PaxExam.class) -@ExamReactorStrategy(PerSuite.class) -@Category(PerSuite.class) -public class DistributedTwoNodesMapReduceTest extends BaseWordCountMapReduceTest { - @Override - protected void createCacheManagers() throws Throwable { - //not used - } - - @Before - public void setUp() { - TestResourceTracker.testThreadStarted(this); - ConfigurationBuilder builder = getDefaultClusteredCacheConfig(getCacheMode(), true); - builder.clustering().stateTransfer().chunkSize(2); - createClusteredCaches(2, cacheName(), builder); - } - - @After - public void tearDown() { - TestingUtil.killCacheManagers(cacheManagers); - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - protected MapReduceTask createMapReduceTask(Cache c){ - //run distributed reduce with per task cache - return new MapReduceTask(c, true, false); - } -} diff --git a/integrationtests/security-manager-it/src/test/java/org/infinispan/security/ExecutionAuthorizationTest.java b/integrationtests/security-manager-it/src/test/java/org/infinispan/security/ExecutionAuthorizationTest.java index 843cb948b864..bc51b9340743 100644 --- a/integrationtests/security-manager-it/src/test/java/org/infinispan/security/ExecutionAuthorizationTest.java +++ b/integrationtests/security-manager-it/src/test/java/org/infinispan/security/ExecutionAuthorizationTest.java @@ -1,27 +1,10 @@ package org.infinispan.security; -import static org.testng.AssertJUnit.assertEquals; - -import java.io.Serializable; -import java.security.Policy; -import java.security.PrivilegedAction; -import java.security.PrivilegedExceptionAction; -import java.util.Iterator; -import java.util.StringTokenizer; -import java.util.concurrent.Callable; - -import javax.security.auth.Subject; - -import org.infinispan.Cache; import org.infinispan.commons.util.concurrent.NotifyingFuture; import org.infinispan.configuration.cache.CacheMode; import org.infinispan.configuration.cache.ConfigurationBuilder; import org.infinispan.configuration.global.GlobalConfigurationBuilder; import org.infinispan.distexec.DefaultExecutorService; -import org.infinispan.distexec.mapreduce.Collector; -import org.infinispan.distexec.mapreduce.MapReduceTask; -import org.infinispan.distexec.mapreduce.Mapper; -import org.infinispan.distexec.mapreduce.Reducer; import org.infinispan.manager.EmbeddedCacheManager; import org.infinispan.security.impl.IdentityRoleMapper; import org.infinispan.test.MultipleCacheManagersTest; @@ -29,6 +12,15 @@ import org.testng.annotations.AfterClass; import org.testng.annotations.Test; +import javax.security.auth.Subject; +import java.io.Serializable; +import java.security.Policy; +import java.security.PrivilegedAction; +import java.security.PrivilegedExceptionAction; +import java.util.concurrent.Callable; + +import static org.testng.AssertJUnit.assertEquals; + /** * ExecutionAuthorizationTest. * @@ -153,68 +145,6 @@ public Void run() throws Exception { } } - private void mapReduceTest() { - Cache c1 = cache(0, EXECUTION_CACHE); - Cache c2 = cache(1, EXECUTION_CACHE); - - c1.put("1", "Hello world here I am"); - c2.put("2", "Infinispan rules the world"); - c1.put("3", "JUDCon is in Boston"); - c2.put("4", "JBoss World is in Boston as well"); - c1.put("12","JBoss Application Server"); - c2.put("15", "Hello world"); - c1.put("14", "Infinispan community"); - - c1.put("111", "Infinispan open source"); - c2.put("112", "Boston is close to Toronto"); - c1.put("113", "Toronto is a capital of Ontario"); - c2.put("114", "JUDCon is cool"); - c1.put("211", "JBoss World is awesome"); - c2.put("212", "JBoss rules"); - c1.put("213", "JBoss division of RedHat "); - c2.put("214", "RedHat community"); - - MapReduceTask task = new MapReduceTask(c1); - task.mappedWith(new WordCountMapper()).reducedWith(new WordCountReducer()); - task.execute(); - } - - public void testExecMapReduce() { - Policy.setPolicy(new SurefireTestingPolicy()); - System.setSecurityManager(new SecurityManager()); - try { - Subject.doAs(EXEC, new PrivilegedAction() { - - @Override - public Void run() { - mapReduceTest(); - return null; - } - }); - } finally { - System.setSecurityManager(null); - Policy.setPolicy(null); - } - } - - @Test(expectedExceptions=SecurityException.class) - public void testNoExecMapReduce() { - Policy.setPolicy(new SurefireTestingPolicy()); - try { - System.setSecurityManager(new SecurityManager()); - Subject.doAs(NOEXEC, new PrivilegedAction() { - - @Override - public Void run() { - mapReduceTest(); - return null; - } - }); - } finally { - System.setSecurityManager(null); - Policy.setPolicy(null); - } - } static class SimpleCallable implements Callable, Serializable { @@ -229,33 +159,4 @@ public Integer call() throws Exception { return 1; } } - static class WordCountMapper implements Mapper { - /** The serialVersionUID */ - private static final long serialVersionUID = -5943370243108735560L; - - @Override - public void map(String key, String value, Collector collector) { - if(value == null) throw new IllegalArgumentException("Key " + key + " has value " + value); - StringTokenizer tokens = new StringTokenizer(value); - while (tokens.hasMoreElements()) { - String s = (String) tokens.nextElement(); - collector.emit(s, 1); - } - } - } - - static class WordCountReducer implements Reducer { - /** The serialVersionUID */ - private static final long serialVersionUID = 1901016598354633256L; - - @Override - public Integer reduce(String key, Iterator iter) { - int sum = 0; - while (iter.hasNext()) { - sum += iter.next(); - } - return sum; - } - } - } diff --git a/scripting/src/main/java/org/infinispan/scripting/impl/CollatorScript.java b/scripting/src/main/java/org/infinispan/scripting/impl/CollatorScript.java deleted file mode 100644 index 65c150491dbc..000000000000 --- a/scripting/src/main/java/org/infinispan/scripting/impl/CollatorScript.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.infinispan.scripting.impl; - -import java.util.Map; - -import javax.script.SimpleBindings; - -import org.infinispan.commons.CacheException; -import org.infinispan.distexec.mapreduce.Collator; - -/** - * CollatorScript. - * - * @author Tristan Tarrant - * @since 7.2 - * @deprecated Use the streaming API within a local script instead - */ -@Deprecated -public class CollatorScript implements Collator { - private final ScriptMetadata metadata; - private final ScriptingManagerImpl scriptManager; - - public CollatorScript(ScriptMetadata metadata, ScriptingManagerImpl scriptManager) { - this.metadata = metadata; - this.scriptManager = scriptManager; - } - - @Override - public R collate(Map reducedResults) { - SimpleBindings bindings = new SimpleBindings(); - bindings.put(SystemBindings.CACHE_MANAGER.toString(), scriptManager.cacheManager); - bindings.put("reducedResults", reducedResults); - try { - return (R) scriptManager.execute(metadata, bindings).get(); - } catch (Exception e) { - throw new CacheException("Error during collator script", e); - } - } -} diff --git a/scripting/src/main/java/org/infinispan/scripting/impl/ExecutionMode.java b/scripting/src/main/java/org/infinispan/scripting/impl/ExecutionMode.java index c14e7c95a301..a1ad231ce808 100644 --- a/scripting/src/main/java/org/infinispan/scripting/impl/ExecutionMode.java +++ b/scripting/src/main/java/org/infinispan/scripting/impl/ExecutionMode.java @@ -8,15 +8,7 @@ */ public enum ExecutionMode { LOCAL(LocalRunner.INSTANCE, false), - DISTRIBUTED(DistributedRunner.INSTANCE, true), - @Deprecated - MAPPER(MapReduceRunner.INSTANCE, true), - @Deprecated - REDUCER(NullRunner.INSTANCE, false), - @Deprecated - COLLATOR(NullRunner.INSTANCE, false), - @Deprecated - COMBINER(NullRunner.INSTANCE, false); + DISTRIBUTED(DistributedRunner.INSTANCE, true); private final ScriptRunner runner; private final boolean clustered; diff --git a/scripting/src/main/java/org/infinispan/scripting/impl/MapReduceRunner.java b/scripting/src/main/java/org/infinispan/scripting/impl/MapReduceRunner.java deleted file mode 100644 index 4a26b7a203bf..000000000000 --- a/scripting/src/main/java/org/infinispan/scripting/impl/MapReduceRunner.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.infinispan.scripting.impl; - -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Future; - -import org.infinispan.Cache; -import org.infinispan.commons.util.concurrent.NotifyingFuture; -import org.infinispan.distexec.mapreduce.MapReduceTask; -import org.infinispan.util.concurrent.CompletableFutures; - -/** - * MapReduceRunner. - * - * @author Tristan Tarrant - * @since 7.2 - * @deprecated Use the streaming API within a local script instead - */ -@Deprecated -public class MapReduceRunner implements ScriptRunner { - public static final MapReduceRunner INSTANCE = new MapReduceRunner(); - - private MapReduceRunner() { - } - - @Override - public CompletableFuture runScript(ScriptingManagerImpl scriptManager, ScriptMetadata metadata, CacheScriptBindings binding) { - MapReduceTask mapReduceTask = new MapReduceTask<>((Cache) binding.get(SystemBindings.CACHE.toString())); - mapReduceTask.mappedWith(new MapperScript<>(metadata)); - metadata.reducer().ifPresent((reducer) -> { - mapReduceTask.reducedWith(new ReducerScript<>(scriptManager.getScriptMetadata(reducer))); - }); - metadata.combiner().ifPresent((combiner) -> { - mapReduceTask.combinedWith(new ReducerScript<>(scriptManager.getScriptMetadata(combiner))); - }); - - Future future = metadata.collator().map((collator) -> { - return mapReduceTask.executeAsynchronously(new CollatorScript<>(scriptManager.getScriptMetadata(collator), scriptManager)); - }).orElseGet(() -> { - return mapReduceTask.executeAsynchronously(); - }); - - return CompletableFutures.toCompletableFuture((NotifyingFuture) future); - } - -} diff --git a/scripting/src/main/java/org/infinispan/scripting/impl/MapperScript.java b/scripting/src/main/java/org/infinispan/scripting/impl/MapperScript.java deleted file mode 100644 index 1ac2b5413258..000000000000 --- a/scripting/src/main/java/org/infinispan/scripting/impl/MapperScript.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.infinispan.scripting.impl; - -import javax.script.Bindings; -import javax.script.SimpleBindings; - -import org.infinispan.commons.marshall.Marshaller; -import org.infinispan.distexec.mapreduce.Collector; -import org.infinispan.distexec.mapreduce.Mapper; -import org.infinispan.manager.EmbeddedCacheManager; -import org.infinispan.scripting.ScriptingManager; - -/** - * MapperScript. - * - * @author Tristan Tarrant - * @since 7.2 - * @deprecated Use the streaming API within a local script instead - */ -@Deprecated -public class MapperScript implements Mapper, EnvironmentAware { - private final ScriptMetadata metadata; - private transient ScriptingManagerImpl scriptManager; - private transient Bindings bindings; - - public MapperScript(ScriptMetadata metadata) { - this.metadata = metadata; - } - - @Override - public void map(KIn key, VIn value, Collector collector) { - bindings.put("key", key); - bindings.put("value", value); - bindings.put("collector", collector); - scriptManager.execute(metadata, bindings); - } - - @Override - public void setEnvironment(EmbeddedCacheManager cacheManager, Marshaller marshaller) { - scriptManager = (ScriptingManagerImpl) SecurityActions.getGlobalComponentRegistry(cacheManager).getComponent(ScriptingManager.class); - bindings = new SimpleBindings(); - bindings.put(SystemBindings.CACHE_MANAGER.toString(), cacheManager); - bindings.put(SystemBindings.MARSHALLER.toString(), marshaller); - } - -} diff --git a/scripting/src/main/java/org/infinispan/scripting/impl/ReducerScript.java b/scripting/src/main/java/org/infinispan/scripting/impl/ReducerScript.java deleted file mode 100644 index a77700d31692..000000000000 --- a/scripting/src/main/java/org/infinispan/scripting/impl/ReducerScript.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.infinispan.scripting.impl; - -import java.util.Iterator; - -import javax.script.SimpleBindings; - -import org.infinispan.commons.CacheException; -import org.infinispan.commons.marshall.Marshaller; -import org.infinispan.distexec.mapreduce.Reducer; -import org.infinispan.manager.EmbeddedCacheManager; -import org.infinispan.scripting.ScriptingManager; - -/** - * ReducerScript. - * - * @author Tristan Tarrant - * @since 7.2 - * @deprecated Use the streaming API within a local script instead - */ -@Deprecated -public class ReducerScript implements Reducer, EnvironmentAware { - private final ScriptMetadata metadata; - private transient ScriptingManagerImpl scriptManager; - private transient SimpleBindings bindings; - - public ReducerScript(ScriptMetadata metadata) { - this.metadata = metadata; - } - - @Override - public VOut reduce(KOut reducedKey, Iterator iter) { - bindings.put("reducedKey", reducedKey); - bindings.put("iter", iter); - try { - return (VOut) scriptManager.execute(metadata, bindings).get(); - } catch (Exception e) { - throw new CacheException("Error during reducer script", e); - } - } - - @Override - public void setEnvironment(EmbeddedCacheManager cacheManager, Marshaller marshaller) { - scriptManager = (ScriptingManagerImpl) SecurityActions.getGlobalComponentRegistry(cacheManager).getComponent(ScriptingManager.class); - bindings = new SimpleBindings(); - bindings.put(SystemBindings.CACHE_MANAGER.toString(), cacheManager); - bindings.put(SystemBindings.MARSHALLER.toString(), marshaller); - } - -} diff --git a/scripting/src/main/java/org/infinispan/scripting/impl/ScriptingMapReduceTaskLifecycle.java b/scripting/src/main/java/org/infinispan/scripting/impl/ScriptingMapReduceTaskLifecycle.java deleted file mode 100644 index 4f744f367976..000000000000 --- a/scripting/src/main/java/org/infinispan/scripting/impl/ScriptingMapReduceTaskLifecycle.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.infinispan.scripting.impl; - -import org.infinispan.Cache; -import org.infinispan.commons.marshall.Marshaller; -import org.infinispan.configuration.cache.CompatibilityModeConfiguration; -import org.infinispan.distexec.mapreduce.Mapper; -import org.infinispan.distexec.mapreduce.Reducer; -import org.infinispan.distexec.mapreduce.spi.MapReduceTaskLifecycle; -import org.infinispan.manager.EmbeddedCacheManager; -import org.kohsuke.MetaInfServices; - -/** - * ScriptingMapReduceTaskLifecycle. - * - * @author Tristan Tarrant - * @since 7.2 - */ -@MetaInfServices -public class ScriptingMapReduceTaskLifecycle implements MapReduceTaskLifecycle { - - @Override - public void onPreExecute(Mapper mapper, Cache inputCache) { - if(mapper instanceof EnvironmentAware) { - applyEnvironment((EnvironmentAware) mapper, inputCache); - } - } - - @Override - public void onPostExecute(Mapper mapper) { - } - - @Override - public void onPreExecute(Reducer reducer, Cache inputCache) { - if(reducer instanceof EnvironmentAware) { - applyEnvironment((EnvironmentAware) reducer, inputCache); - } - } - - @Override - public void onPostExecute(Reducer reducer) { - } - - private void applyEnvironment(EnvironmentAware mapper, Cache cache) { - EmbeddedCacheManager cacheManager = cache.getCacheManager(); - CompatibilityModeConfiguration compatibility = cache.getCacheConfiguration().compatibility(); - Marshaller marshaller = compatibility.enabled() ? compatibility.marshaller() : cacheManager.getCacheManagerConfiguration().serialization().marshaller(); - mapper.setEnvironment(cacheManager, marshaller); - } -} diff --git a/scripting/src/test/java/org/infinispan/scripting/ClusteredScriptingTest.java b/scripting/src/test/java/org/infinispan/scripting/ClusteredScriptingTest.java index d2843a7e48ca..710a91649e25 100644 --- a/scripting/src/test/java/org/infinispan/scripting/ClusteredScriptingTest.java +++ b/scripting/src/test/java/org/infinispan/scripting/ClusteredScriptingTest.java @@ -87,27 +87,4 @@ public void testDistributedMapReduceStream() throws IOException, ExecutionExcept assertEquals(resultsFuture.get(0).get("macbeth"), Long.valueOf(287)); assertEquals(resultsFuture.get(1).get("macbeth"), Long.valueOf(287)); } - - public void testMapReduce() throws Exception { - ScriptingManager scriptingManager = getScriptingManager(manager(0)); - Cache cache = cache(0); - loadData(cache, "/macbeth.txt"); - loadScript(scriptingManager, "/wordCountMapper.js"); - loadScript(scriptingManager, "/wordCountReducer.js"); - loadScript(scriptingManager, "/wordCountCollator.js"); - CompletableFuture future = scriptingManager.runScript("wordCountMapper.js", new TaskContext().cache(cache)); - LinkedHashMap results = (LinkedHashMap)future.get(); - assertEquals(20, results.size()); - assertTrue(results.get("macbeth").equals(Double.valueOf(287))); - } - - @Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = ".*cannot be invoked directly since it specifies mode 'REDUCER'") - public void testOnlyReduceTaskRun() throws Exception { - ScriptingManager scriptingManager = getScriptingManager(manager(0)); - Cache cache = cache(0); - loadData(cache, "/macbeth.txt"); - loadScript(scriptingManager, "/wordCountReducer.js"); - - scriptingManager.runScript("wordCountReducer.js", new TaskContext().cache(cache)); - } } diff --git a/scripting/src/test/resources/wordCountCollator.js b/scripting/src/test/resources/wordCountCollator.js deleted file mode 100644 index 20eab2988dc2..000000000000 --- a/scripting/src/test/resources/wordCountCollator.js +++ /dev/null @@ -1,12 +0,0 @@ -// mode=collator,language=javascript -var entrySet = reducedResults.entrySet(); -var l = new java.util.ArrayList(entrySet); -java.util.Collections.sort(l, new org.infinispan.scripting.EntryComparator()) - -var results = new java.util.LinkedHashMap(); -for (var i=0; i<20; i++) { - var entry = l.get(i); - results.put(entry.getKey(), entry.getValue()); -} - -results diff --git a/scripting/src/test/resources/wordCountMapper.js b/scripting/src/test/resources/wordCountMapper.js deleted file mode 100644 index 78b1c55d5d51..000000000000 --- a/scripting/src/test/resources/wordCountMapper.js +++ /dev/null @@ -1,9 +0,0 @@ -// mode=mapper,reducer=wordCountReducer.js,collator=wordCountCollator.js,language=javascript -var re = /[\W]+/ -var words = value.split(re) -for (var i=0; i < words.length; i++) { - var word = words[i]; - if (word != null && word.length > 5) { - collector.emit(words[i].toLowerCase(), 1); - } -} diff --git a/scripting/src/test/resources/wordCountReducer.js b/scripting/src/test/resources/wordCountReducer.js deleted file mode 100644 index 88e361413e64..000000000000 --- a/scripting/src/test/resources/wordCountReducer.js +++ /dev/null @@ -1,6 +0,0 @@ -// mode=reducer,language=javascript -var sum = 0; -while (iter.hasNext()) { - sum += parseInt(iter.next()); -} -sum diff --git a/server/hotrod/src/main/scala/org/infinispan/server/hotrod/AbstractEncoder1x.scala b/server/hotrod/src/main/scala/org/infinispan/server/hotrod/AbstractEncoder1x.scala index d11faae0ae51..1f29c2f38986 100644 --- a/server/hotrod/src/main/scala/org/infinispan/server/hotrod/AbstractEncoder1x.scala +++ b/server/hotrod/src/main/scala/org/infinispan/server/hotrod/AbstractEncoder1x.scala @@ -111,8 +111,7 @@ abstract class AbstractEncoder1x extends AbstractVersionedEncoder with Constants log.trace("About to respond to bulk get keys request") if (g.status == Success) { val cache: Cache = server.getCacheInstance(g.cacheName, cacheManager, false) - val keys = BulkUtil.getAllKeys(cache, g.scope) - val iterator = asScalaIterator(keys.iterator) + val iterator = asScalaIterator(BulkUtil.getAllKeys(cache, g.scope)) for (key <- iterator) { buf.writeByte(1) // Not done writeRangedBytes(key, buf) diff --git a/server/hotrod/src/main/scala/org/infinispan/server/hotrod/Encoder2x.scala b/server/hotrod/src/main/scala/org/infinispan/server/hotrod/Encoder2x.scala index 7a5b38c240c4..c12ecb4c8671 100644 --- a/server/hotrod/src/main/scala/org/infinispan/server/hotrod/Encoder2x.scala +++ b/server/hotrod/src/main/scala/org/infinispan/server/hotrod/Encoder2x.scala @@ -298,8 +298,7 @@ object Encoder2x extends AbstractVersionedEncoder with Constants with Log { if (isTrace) log.trace("About to respond to bulk get keys request") if (g.status == Success) { val cache: Cache = server.getCacheInstance(g.cacheName, cacheManager, false) - val keys = BulkUtil.getAllKeys(cache, g.scope) - val iterator = asScalaIterator(keys.iterator) + val iterator = asScalaIterator(BulkUtil.getAllKeys(cache, g.scope)) for (key <- iterator) { buf.writeByte(1) // Not done writeRangedBytes(key, buf) diff --git a/server/hotrod/src/main/scala/org/infinispan/server/hotrod/util/BulkUtil.java b/server/hotrod/src/main/scala/org/infinispan/server/hotrod/util/BulkUtil.java index d50eb562c33d..5c17370f5804 100644 --- a/server/hotrod/src/main/scala/org/infinispan/server/hotrod/util/BulkUtil.java +++ b/server/hotrod/src/main/scala/org/infinispan/server/hotrod/util/BulkUtil.java @@ -1,22 +1,19 @@ package org.infinispan.server.hotrod.util; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; - import org.infinispan.Cache; +import org.infinispan.CacheStream; +import org.infinispan.commons.CacheException; import org.infinispan.commons.marshall.Marshaller; -import org.infinispan.commons.util.Immutables; -import org.infinispan.configuration.cache.CacheMode; +import org.infinispan.commons.marshall.StreamingMarshaller; +import org.infinispan.commons.util.IteratorMapper; import org.infinispan.configuration.cache.CompatibilityModeConfiguration; -import org.infinispan.distexec.mapreduce.Collator; -import org.infinispan.distexec.mapreduce.Collector; -import org.infinispan.distexec.mapreduce.MapReduceTask; -import org.infinispan.distexec.mapreduce.Mapper; -import org.infinispan.distexec.mapreduce.Reducer; +import org.infinispan.factories.KnownComponentNames; import org.infinispan.server.hotrod.HotRodTypeConverter; +import java.io.IOException; +import java.util.Iterator; +import java.util.Set; + /** * @author Ray Tsang * @since 5.2 @@ -29,73 +26,18 @@ public final class BulkUtil { public static final int GLOBAL_SCOPE = 1; public static final int LOCAL_SCOPE = 2; - public static Set getAllKeys(Cache cache, int scope) { - CacheMode cacheMode = cache.getAdvancedCache().getCacheConfiguration().clustering().cacheMode(); - boolean keysAreLocal = !cacheMode.isClustered() || cacheMode.isReplicated(); - if (keysAreLocal || scope == LOCAL_SCOPE) { - return cache.keySet(); - } else { - MapReduceTask task = - new MapReduceTask((Cache) cache) - .mappedWith(new KeyMapper()) - .reducedWith(new KeyReducer()); - return task.execute(createCollator(cache)); - } - } - - private static Collator> createCollator(Cache cache) { + public static Iterator getAllKeys(Cache cache, int scope) { CompatibilityModeConfiguration compatibility = cache.getCacheConfiguration().compatibility(); - boolean enabled = compatibility.enabled(); - return enabled ? new CompatibilityCollator(compatibility.marshaller()) - : new KeysCollator(); - } - - private static class KeyMapper implements Mapper { - - private static final long serialVersionUID = -5054573988280497412L; - - @Override - public void map(K key, Object value, Collector collector) { - collector.emit(key, null); + CacheStream stream = cache.keySet().stream(); + HotRodTypeConverter converter = new HotRodTypeConverter(); + if (compatibility.enabled() && compatibility.marshaller() != null) { + converter.setMarshaller(compatibility.marshaller()); } + return new IteratorMapper(stream.iterator(), k -> { + if (k instanceof byte[]) { + return (byte[]) k; + } + return (byte[]) converter.unboxKey(k); + }); } - - private static class KeyReducer implements Reducer { - - private static final long serialVersionUID = -8199097945001793869L; - - @Override - public Object reduce(K reducedKey, Iterator iter) { - return null; // the value is not actually used, we can output null - } - } - - private static class KeysCollator implements Collator> { - - @Override - public Set collate(Map reducedResults) { - return reducedResults.keySet(); - } - } - - private static class CompatibilityCollator implements Collator> { - - private final HotRodTypeConverter converter = new HotRodTypeConverter(); - - private CompatibilityCollator(Marshaller compatibilityMarshaller) { - if (compatibilityMarshaller != null) - converter.setMarshaller(compatibilityMarshaller); - } - - @Override - public Set collate(Map reducedResults) { - Set keySet = reducedResults.keySet(); - Set backingSet = new HashSet(keySet.size()); - for (K key : keySet) - backingSet.add((K) converter.unboxKey(key)); - - return Immutables.immutableSetWrap(backingSet); - } - } - } diff --git a/server/integration/testsuite/src/test/resources/stream_serverTask.js b/server/integration/testsuite/src/test/resources/stream_serverTask.js index a2a0ba5c0287..3f93f7569bcf 100644 --- a/server/integration/testsuite/src/test/resources/stream_serverTask.js +++ b/server/integration/testsuite/src/test/resources/stream_serverTask.js @@ -5,7 +5,7 @@ var Collectors = Java.type("java.util.stream.Collectors") var Arrays = Java.type("org.infinispan.scripting.utils.JSArrays") var CacheCollectors = Java.type("org.infinispan.stream.CacheCollectors"); var SerializableFunction = Java.extend(Function, Serializable); -var SerializableSupplier = Java.extend(Java.type("org.infinispan.util.SerializableSupplier")) +var SerializableSupplier = Java.extend(Java.type("org.infinispan.util.function.SerializableSupplier")) var e = new SerializableFunction( { apply: function(object) {