From 0054c0d3e31854a9633a7a8601fdf4c1ca266d47 Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Wed, 14 Jan 2015 11:10:25 +0400 Subject: [PATCH 01/36] IGNITE-53: Add distributed Cache.iterator --- .../processors/cache/IgniteCacheProxy.java | 66 ++++++++++++++++++- .../cache/IgniteCacheProxyApiTest.java | 7 ++ 2 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/IgniteCacheProxyApiTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index a985fde5821d4..67797cfa4a26e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -14,6 +14,8 @@ import org.apache.ignite.cache.query.*; import org.apache.ignite.lang.*; import org.gridgain.grid.cache.*; +import org.gridgain.grid.cache.query.GridCacheQuery; +import org.gridgain.grid.cache.query.GridCacheQueryFuture; import org.gridgain.grid.kernal.*; import org.gridgain.grid.kernal.processors.cache.*; import org.gridgain.grid.util.tostring.*; @@ -777,9 +779,9 @@ public void removeAll(Collection keys) { } /** {@inheritDoc} */ - @Override public Iterator> iterator() { - // TODO IGNITE-1. - throw new UnsupportedOperationException(); + @Override + public Iterator> iterator() { + return new IgniteCacheIterator(); } /** {@inheritDoc} */ @@ -922,4 +924,62 @@ private CacheException cacheException(IgniteCheckedException e) { @Override public String toString() { return S.toString(IgniteCacheProxy.class, this); } + + /** + * Iterator over the IgniteCacheProxy + */ + private class IgniteCacheIterator implements Iterator> { + + /** Cache query future for all entries in distributed ignite cache. */ + private final GridCacheQueryFuture> fut; + + /** Current element from all entries in distributed ignite cache. */ + private Map.Entry curIter; + + public IgniteCacheIterator() { + fut = delegate.queries().createScanQuery(null).execute(); + } + + /** {@inheritDoc} */ + @Override public boolean hasNext() { + try { + curIter = fut.next(); + return curIter != null; + } catch (IgniteCheckedException e) { + e.printStackTrace(); + //TODO: ???? + } + return false; + } + + /** {@inheritDoc} */ + @Override public Entry next() { + return new Cache.Entry() { + /** {@inheritDoc} */ + @Override public K getKey() { + return curIter.getKey(); + } + + /** {@inheritDoc} */ + @Override public V getValue() { + return curIter.getValue(); + } + + /** {@inheritDoc} */ + @Override public T unwrap(Class clazz) { + throw new IllegalArgumentException(); + } + }; + } + + /** {@inheritDoc} */ + @Override public void remove() { + try { + delegate.remove(curIter.getKey(), curIter.getValue()); + } catch (IgniteCheckedException e) { + //TODO: ??? + e.printStackTrace(); + } + } + } } diff --git a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/IgniteCacheProxyApiTest.java b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/IgniteCacheProxyApiTest.java new file mode 100644 index 0000000000000..17c4c9d53bf75 --- /dev/null +++ b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/IgniteCacheProxyApiTest.java @@ -0,0 +1,7 @@ +package org.gridgain.grid.kernal.processors.cache; + +/** + * Created by GridGain on 13.01.2015. + */ +public class IgniteCacheProxyApiTest { +} From 8733b7c98cdf238151f92ee387f6c7dc6ce27af5 Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Wed, 14 Jan 2015 13:11:30 +0400 Subject: [PATCH 02/36] Add tests for iterator --- .../processors/cache/IgniteCacheProxy.java | 7 +-- .../GridCacheAbstractFullApiSelfTest.java | 46 +++++++++++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index 67797cfa4a26e..333cb33ce2290 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -946,10 +946,8 @@ public IgniteCacheIterator() { curIter = fut.next(); return curIter != null; } catch (IgniteCheckedException e) { - e.printStackTrace(); - //TODO: ???? + throw cacheException(e); } - return false; } /** {@inheritDoc} */ @@ -977,8 +975,7 @@ public IgniteCacheIterator() { try { delegate.remove(curIter.getKey(), curIter.getValue()); } catch (IgniteCheckedException e) { - //TODO: ??? - e.printStackTrace(); + throw cacheException(e); } } } diff --git a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java index 54397cc62cc5d..504a435ccc1a9 100644 --- a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java +++ b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java @@ -24,6 +24,7 @@ import org.gridgain.testframework.*; import org.jetbrains.annotations.*; +import javax.cache.Cache; import javax.cache.expiry.*; import javax.cache.processor.*; import java.util.*; @@ -5240,4 +5241,49 @@ protected List primaryKeysForCache(GridCacheProjection throw new IgniteCheckedException("Unable to find " + cnt + " keys as primary for cache."); } + + /** + * @throws Exception If failed. + */ + public void testIgniteCacheIterator() throws Exception { + IgniteCache cache = jcache(0); + for (int i = 0; i < gridCount(); ++i) { + cache.put(Integer.toString(i), i); + } + + checkIteratorCacheSize(cache, gridCount()); + + removeCacheIterator(cache); + + checkIteratorCacheSize(cache, gridCount() - 1); + } + + /** + * Remove one element from the cache. Throws exception if cache is empty. + * @param cache Cache. + * @throws Exception + */ + private void removeCacheIterator(IgniteCache cache) throws Exception { + Iterator> iter = cache.iterator(); + if (iter.hasNext()) { + iter.remove(); + } else { + assert false; + } + } + + /** + * @param cache Cache. + * @param size Expected value of cache's size. + * @throws Exception if iteration size is not equal to expected value + */ + private void checkIteratorCacheSize(IgniteCache cache, int size) throws Exception { + Iterator> iter = cache.iterator(); + int cnt = 0; + while (iter.hasNext()) { + iter.next(); + cnt++; + } + assert cnt == size; + } } From 84597ce4d6ffa235c1f9ce3444d2874de9319df5 Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Wed, 14 Jan 2015 13:15:57 +0400 Subject: [PATCH 03/36] Coding guidelines --- .../ignite/internal/processors/cache/IgniteCacheProxy.java | 6 ++++-- .../processors/cache/GridCacheAbstractFullApiSelfTest.java | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index 333cb33ce2290..c6b4aba9ce306 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -945,7 +945,8 @@ public IgniteCacheIterator() { try { curIter = fut.next(); return curIter != null; - } catch (IgniteCheckedException e) { + } + catch (IgniteCheckedException e) { throw cacheException(e); } } @@ -974,7 +975,8 @@ public IgniteCacheIterator() { @Override public void remove() { try { delegate.remove(curIter.getKey(), curIter.getValue()); - } catch (IgniteCheckedException e) { + } + catch (IgniteCheckedException e) { throw cacheException(e); } } diff --git a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java index 504a435ccc1a9..4b835321fd06e 100644 --- a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java +++ b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java @@ -5260,6 +5260,7 @@ public void testIgniteCacheIterator() throws Exception { /** * Remove one element from the cache. Throws exception if cache is empty. + * * @param cache Cache. * @throws Exception */ @@ -5267,7 +5268,8 @@ private void removeCacheIterator(IgniteCache cache) throws Exce Iterator> iter = cache.iterator(); if (iter.hasNext()) { iter.remove(); - } else { + } + else { assert false; } } From 39946d07698c1493be92192527ed10e7aa16a5c5 Mon Sep 17 00:00:00 2001 From: sboikov Date: Wed, 14 Jan 2015 14:43:20 +0300 Subject: [PATCH 04/36] # ignite-53 review --- .../processors/cache/IgniteCacheProxy.java | 29 ++++++++++------- .../GridCacheAbstractFullApiSelfTest.java | 31 ++++++++++--------- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index 307bb054f3e1d..7e2020421404f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -16,8 +16,7 @@ import org.apache.ignite.lang.*; import org.apache.ignite.resources.*; import org.gridgain.grid.cache.*; -import org.gridgain.grid.cache.query.GridCacheQuery; -import org.gridgain.grid.cache.query.GridCacheQueryFuture; +import org.gridgain.grid.cache.query.*; import org.gridgain.grid.kernal.*; import org.gridgain.grid.kernal.processors.cache.*; import org.gridgain.grid.util.tostring.*; @@ -852,9 +851,15 @@ public void removeAll(Collection keys) { } /** {@inheritDoc} */ - @Override - public Iterator> iterator() { - return new IgniteCacheIterator(); + @Override public Iterator> iterator() { + GridCacheProjectionImpl prev = gate.enter(prj); + + try { + return new IgniteCacheIterator(); + } + finally { + gate.leave(prev); + } } /** {@inheritDoc} */ @@ -1069,16 +1074,18 @@ private LoadCacheClosure(String cacheName, IgniteBiPredicate p, Object[] a } /** - * Iterator over the IgniteCacheProxy + * Iterator over the cache. */ private class IgniteCacheIterator implements Iterator> { - /** Cache query future for all entries in distributed ignite cache. */ private final GridCacheQueryFuture> fut; - /** Current element from all entries in distributed ignite cache. */ + /** Current element. */ private Map.Entry curIter; + /** + * + */ public IgniteCacheIterator() { fut = delegate.queries().createScanQuery(null).execute(); } @@ -1087,6 +1094,7 @@ public IgniteCacheIterator() { @Override public boolean hasNext() { try { curIter = fut.next(); + return curIter != null; } catch (IgniteCheckedException e) { @@ -1097,17 +1105,14 @@ public IgniteCacheIterator() { /** {@inheritDoc} */ @Override public Entry next() { return new Cache.Entry() { - /** {@inheritDoc} */ @Override public K getKey() { return curIter.getKey(); } - /** {@inheritDoc} */ @Override public V getValue() { return curIter.getValue(); } - /** {@inheritDoc} */ @Override public T unwrap(Class clazz) { throw new IllegalArgumentException(); } @@ -1117,7 +1122,7 @@ public IgniteCacheIterator() { /** {@inheritDoc} */ @Override public void remove() { try { - delegate.remove(curIter.getKey(), curIter.getValue()); + delegate.removex(curIter.getKey()); } catch (IgniteCheckedException e) { throw cacheException(e); diff --git a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java index a6e02dd39d9f0..e2d7ba593d23a 100644 --- a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java +++ b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java @@ -24,7 +24,7 @@ import org.gridgain.testframework.*; import org.jetbrains.annotations.*; -import javax.cache.Cache; +import javax.cache.*; import javax.cache.expiry.*; import javax.cache.processor.*; import java.util.*; @@ -5247,45 +5247,46 @@ protected List primaryKeysForCache(GridCacheProjection */ public void testIgniteCacheIterator() throws Exception { IgniteCache cache = jcache(0); - for (int i = 0; i < gridCount(); ++i) { + + for (int i = 0; i < 100; ++i) cache.put(Integer.toString(i), i); - } - checkIteratorCacheSize(cache, gridCount()); + checkIteratorCacheSize(cache, 100); removeCacheIterator(cache); - checkIteratorCacheSize(cache, gridCount() - 1); + checkIteratorCacheSize(cache, 100 - 1); } /** * Remove one element from the cache. Throws exception if cache is empty. * * @param cache Cache. - * @throws Exception */ - private void removeCacheIterator(IgniteCache cache) throws Exception { + private void removeCacheIterator(IgniteCache cache) { Iterator> iter = cache.iterator(); - if (iter.hasNext()) { + + if (iter.hasNext()) iter.remove(); - } - else { - assert false; - } + else + fail(); } /** * @param cache Cache. * @param size Expected value of cache's size. - * @throws Exception if iteration size is not equal to expected value */ - private void checkIteratorCacheSize(IgniteCache cache, int size) throws Exception { + private void checkIteratorCacheSize(IgniteCache cache, int size) { Iterator> iter = cache.iterator(); + int cnt = 0; + while (iter.hasNext()) { iter.next(); + cnt++; } - assert cnt == size; + + assertEquals(size, cnt); } } From 8a8182a4b6ad775cc6f87dc1f51757aac7ee81b9 Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Wed, 14 Jan 2015 14:47:03 +0400 Subject: [PATCH 05/36] delete file --- .../kernal/processors/cache/IgniteCacheProxyApiTest.java | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/IgniteCacheProxyApiTest.java diff --git a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/IgniteCacheProxyApiTest.java b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/IgniteCacheProxyApiTest.java deleted file mode 100644 index 17c4c9d53bf75..0000000000000 --- a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/IgniteCacheProxyApiTest.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.gridgain.grid.kernal.processors.cache; - -/** - * Created by GridGain on 13.01.2015. - */ -public class IgniteCacheProxyApiTest { -} From 984ac18cd199a8b69adcb4e7b7e1078530bd3bb1 Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Wed, 14 Jan 2015 16:40:24 +0400 Subject: [PATCH 06/36] Fix Cache.Iterator and add tests to it --- .../processors/cache/IgniteCacheProxy.java | 36 +++++-- .../GridCacheAbstractFullApiSelfTest.java | 95 ++++++++++++++++--- 2 files changed, 110 insertions(+), 21 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index 7e2020421404f..3d8c75cc0c0ff 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -1083,27 +1083,45 @@ private class IgniteCacheIterator implements Iterator> { /** Current element. */ private Map.Entry curIter; + /** Next element. */ + private Map.Entry nextIter; + /** - * + * No-arg constructor. */ public IgniteCacheIterator() { fut = delegate.queries().createScanQuery(null).execute(); - } - /** {@inheritDoc} */ - @Override public boolean hasNext() { try { - curIter = fut.next(); - - return curIter != null; + nextIter = fut.next(); } catch (IgniteCheckedException e) { throw cacheException(e); } } + /** {@inheritDoc} */ + @Override public boolean hasNext() { + return nextIter != null; + } + /** {@inheritDoc} */ @Override public Entry next() { + curIter = nextIter; + + if (curIter == null) { + throw new NoSuchElementException(); + } + + try { + nextIter = fut.next(); + } + catch (IgniteCheckedException e) { + curIter = null; + + throw cacheException(e); + } + return new Cache.Entry() { @Override public K getKey() { return curIter.getKey(); @@ -1121,12 +1139,16 @@ public IgniteCacheIterator() { /** {@inheritDoc} */ @Override public void remove() { + if (curIter == null) + throw new IllegalStateException(); + try { delegate.removex(curIter.getKey()); } catch (IgniteCheckedException e) { throw cacheException(e); } + curIter = null; } } } diff --git a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java index e2d7ba593d23a..0753ab489f3a3 100644 --- a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java +++ b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java @@ -5248,45 +5248,112 @@ protected List primaryKeysForCache(GridCacheProjection public void testIgniteCacheIterator() throws Exception { IgniteCache cache = jcache(0); - for (int i = 0; i < 100; ++i) + final int cacheSz = 100; + Map entries = new HashMap(); + + for (int i = 0; i < cacheSz; ++i) { cache.put(Integer.toString(i), i); + entries.put(Integer.toString(i), i); + } + + checkIteratorHasNext(); - checkIteratorCacheSize(cache, 100); + checkIteratorCache(entries); - removeCacheIterator(cache); + checkIteratorRemove(cache, entries); - checkIteratorCacheSize(cache, 100 - 1); } /** - * Remove one element from the cache. Throws exception if cache is empty. - * + * If hasNext() is called repeatedly, it should return the same result. + */ + private void checkIteratorHasNext() { + Iterator> iter = jcache(0).iterator(); + + assertEquals(iter.hasNext(), iter.hasNext()); + + while (iter.hasNext()) + iter.next(); + + assertFalse(iter.hasNext()); + } + + /** * @param cache Cache. + * @param entries Expected entries in the cache. */ - private void removeCacheIterator(IgniteCache cache) { - Iterator> iter = cache.iterator(); + private void checkIteratorRemove(IgniteCache cache, Map entries) { + //Check that we can remove element. + String rmvKey = Integer.toString(5); + removeCacheIterator(cache, rmvKey); + entries.remove(rmvKey); + assertFalse(cache.containsKey(rmvKey)); + + checkIteratorCache(entries); + + try { + //check that we cannot call Iterator.remove() without next(). + Iterator> iter = jcache(0).iterator(); + + assertTrue(iter.hasNext()); - if (iter.hasNext()) + iter.next(); iter.remove(); - else + iter.remove(); + fail(); + } + catch (Exception e) { + } } /** * @param cache Cache. - * @param size Expected value of cache's size. + * @param key Key to remove. */ - private void checkIteratorCacheSize(IgniteCache cache, int size) { + private void removeCacheIterator(IgniteCache cache, String key) { + Iterator> iter = cache.iterator(); + int delCnt = 0; + + while (iter.hasNext()) { + Cache.Entry cur = iter.next(); + + if (cur.getKey().equals(key)) { + iter.remove(); + delCnt++; + } + + } + + assertEquals(1, delCnt); + } + + /** + * @param entries Expected entries in the cache. + */ + private void checkIteratorCache(Map entries) { + for (int i = 0; i < gridCount(); ++i) + checkIteratorCache(jcache(i), entries); + } + + /** + * @param cache Cache. + * @param entries Expected entries in the cache. + */ + private void checkIteratorCache(IgniteCache cache, Map entries) { Iterator> iter = cache.iterator(); int cnt = 0; while (iter.hasNext()) { - iter.next(); + Cache.Entry cur = iter.next(); + + assertTrue(entries.containsKey(cur.getKey())); + assertEquals(entries.get(cur.getKey()), cur.getValue()); cnt++; } - assertEquals(size, cnt); + assertEquals(entries.size(), cnt); } } From adf6bf9f9ea810beb6e77dde5bd3f36756ba2ea2 Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Thu, 15 Jan 2015 11:27:52 +0400 Subject: [PATCH 07/36] #IGNITE-53: wip --- .../processors/cache/IgniteCacheProxy.java | 66 +++++++++++-------- .../GridCacheAbstractFullApiSelfTest.java | 15 +++-- 2 files changed, 47 insertions(+), 34 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index 3d8c75cc0c0ff..6a75ebba23547 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -1078,57 +1078,49 @@ private LoadCacheClosure(String cacheName, IgniteBiPredicate p, Object[] a */ private class IgniteCacheIterator implements Iterator> { /** Cache query future for all entries in distributed ignite cache. */ - private final GridCacheQueryFuture> fut; + private GridCacheQueryFuture> fut; /** Current element. */ - private Map.Entry curIter; + private Map.Entry curEntry; /** Next element. */ - private Map.Entry nextIter; + private Map.Entry nextEntry; - /** - * No-arg constructor. - */ - public IgniteCacheIterator() { - fut = delegate.queries().createScanQuery(null).execute(); - - try { - nextIter = fut.next(); - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } + /** Init first time. */ + private boolean firstTime = true; /** {@inheritDoc} */ @Override public boolean hasNext() { - return nextIter != null; + initFirstTime(); + + return nextEntry != null; } /** {@inheritDoc} */ @Override public Entry next() { - curIter = nextIter; + initFirstTime(); - if (curIter == null) { + curEntry = nextEntry; + + if (curEntry == null) throw new NoSuchElementException(); - } try { - nextIter = fut.next(); + nextEntry = fut.next(); } catch (IgniteCheckedException e) { - curIter = null; + curEntry = null; throw cacheException(e); } return new Cache.Entry() { @Override public K getKey() { - return curIter.getKey(); + return curEntry.getKey(); } @Override public V getValue() { - return curIter.getValue(); + return curEntry.getValue(); } @Override public T unwrap(Class clazz) { @@ -1139,16 +1131,36 @@ public IgniteCacheIterator() { /** {@inheritDoc} */ @Override public void remove() { - if (curIter == null) + if (curEntry == null) throw new IllegalStateException(); try { - delegate.removex(curIter.getKey()); + delegate.removex(curEntry.getKey()); + } + catch (IgniteCheckedException e) { + throw cacheException(e); + } + + curEntry = null; + } + + /** + * Initialize fields at first call + */ + private void initFirstTime() { + if (!firstTime) { + return; + } + + firstTime = false; + fut = delegate.queries().createScanQuery(null).execute(); + + try { + nextEntry = fut.next(); } catch (IgniteCheckedException e) { throw cacheException(e); } - curIter = null; } } } diff --git a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java index 0753ab489f3a3..2cf58608b0808 100644 --- a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java +++ b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java @@ -5291,19 +5291,20 @@ private void checkIteratorRemove(IgniteCache cache, Map> iter = jcache(0).iterator(); + //check that we cannot call Iterator.remove() without next(). + Iterator> iter = jcache(0).iterator(); - assertTrue(iter.hasNext()); + assertTrue(iter.hasNext()); - iter.next(); - iter.remove(); + iter.next(); + iter.remove(); + + try { iter.remove(); fail(); } - catch (Exception e) { + catch (IllegalStateException e) { } } From b4c2ca14ddc823d2546efdf52ca3cb6b078c9865 Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Thu, 15 Jan 2015 15:32:57 +0400 Subject: [PATCH 08/36] #IGNITE-53: Refactor out the common part from IgniteCacheProxy and GridCacheSetImpl --- .../processors/cache/IgniteCacheProxy.java | 238 +++++++++++++----- .../IgniteQueryFutureStorage.java | 192 ++++++++++++++ 2 files changed, 371 insertions(+), 59 deletions(-) create mode 100644 modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/IgniteQueryFutureStorage.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index e0cfa5ab0312f..c41f44eeb41d6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -27,6 +27,7 @@ import org.gridgain.grid.cache.query.*; import org.gridgain.grid.kernal.*; import org.gridgain.grid.kernal.processors.cache.*; +import org.gridgain.grid.kernal.processors.cache.datastructures.*; import org.gridgain.grid.util.tostring.*; import org.gridgain.grid.util.typedef.*; import org.gridgain.grid.util.typedef.internal.*; @@ -62,6 +63,9 @@ public class IgniteCacheProxy extends IgniteAsyncSupportAdapter implements /** Projection. */ private GridCacheProjectionImpl prj; + /** Query future storage */ + private final IgniteQueryFutureStorage queryStorage; + /** * @param ctx Context. * @param delegate Delegate. @@ -81,6 +85,8 @@ public IgniteCacheProxy(GridCacheContext ctx, this.delegate = delegate; this.prj = prj; + this.queryStorage = new IgniteQueryFutureStorage(ctx); + gate = ctx.gate(); } @@ -88,6 +94,8 @@ public IgniteCacheProxy(GridCacheContext ctx, * @return Context. */ public GridCacheContext context() { + onAccess(); + return ctx; } @@ -95,11 +103,15 @@ public GridCacheContext context() { * @return Ignite instance. */ @Override public GridEx ignite() { + onAccess(); + return ctx.grid(); } /** {@inheritDoc} */ @Override public > C getConfiguration(Class clazz) { + onAccess(); + GridCacheConfiguration cfg = ctx.config(); if (!clazz.isAssignableFrom(cfg.getClass())) @@ -110,12 +122,16 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Nullable @Override public Entry randomEntry() { + onAccess(); + // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public IgniteCache withExpiryPolicy(ExpiryPolicy plc) { + onAccess(); + GridCacheProjectionImpl prev = gate.enter(prj); try { @@ -130,6 +146,8 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public void loadCache(@Nullable IgniteBiPredicate p, @Nullable Object... args) { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -151,6 +169,8 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public void localLoadCache(@Nullable IgniteBiPredicate p, @Nullable Object... args) { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -168,6 +188,8 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Nullable @Override public V getAndPutIf(K key, V val, IgnitePredicate> filter) { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -185,6 +207,8 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public boolean putIf(K key, V val, IgnitePredicate> filter) { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -202,6 +226,8 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public V getAndRemoveIf(K key, IgnitePredicate> filter) { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -219,6 +245,8 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public boolean removeIf(K key, IgnitePredicate> filter) { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -236,6 +264,8 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Nullable @Override public V getAndPutIfAbsent(K key, V val) throws CacheException { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -253,24 +283,32 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public void removeAll(IgnitePredicate filter) throws CacheException { + onAccess(); + // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public Lock lock(K key) throws CacheException { + onAccess(); + // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public Lock lockAll(Set keys) throws CacheException { + onAccess(); + // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public boolean isLocked(K key) { + onAccess(); + GridCacheProjectionImpl prev = gate.enter(prj); try { @@ -283,6 +321,8 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public boolean isLockedByThread(K key) { + onAccess(); + GridCacheProjectionImpl prev = gate.enter(prj); try { @@ -295,18 +335,24 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public Iterable> localEntries(CachePeekMode... peekModes) throws CacheException { + onAccess(); + // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public Map localPartition(int part) throws CacheException { + onAccess(); + // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public void localEvict(Collection keys) { + onAccess(); + GridCacheProjectionImpl prev = gate.enter(prj); try { @@ -319,6 +365,8 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Nullable @Override public V localPeek(K key, CachePeekMode... peekModes) { + onAccess(); + // TODO IGNITE-1. if (peekModes.length != 0) throw new UnsupportedOperationException(); @@ -335,6 +383,8 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public void localPromote(Set keys) throws CacheException { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -352,12 +402,16 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public boolean clear(Collection keys) { + onAccess(); + // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public int size(CachePeekMode... peekModes) throws CacheException { + onAccess(); + // TODO IGNITE-1. if (peekModes.length != 0) throw new UnsupportedOperationException(); @@ -374,6 +428,8 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public int localSize(CachePeekMode... peekModes) { + onAccess(); + // TODO IGNITE-1. GridCacheProjectionImpl prev = gate.enter(prj); @@ -387,6 +443,8 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public V get(K key) { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -404,6 +462,8 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public Map getAll(Set keys) { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -424,6 +484,8 @@ public GridCacheContext context() { * @return Values map. */ public Map getAll(Collection keys) { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -446,6 +508,8 @@ public Map getAll(Collection keys) { * @return Entry set. */ public Set> entrySetx(IgnitePredicate>... filter) { + onAccess(); + GridCacheProjectionImpl prev = gate.enter(prj); try { @@ -460,6 +524,8 @@ public Set> entrySetx(IgnitePredicate> * @param filter Filter. */ public void removeAll(IgnitePredicate>... filter) { + onAccess(); + GridCacheProjectionImpl prev = gate.enter(prj); try { @@ -475,6 +541,8 @@ public void removeAll(IgnitePredicate>... filter) { /** {@inheritDoc} */ @Override public boolean containsKey(K key) { + onAccess(); + GridCacheProjectionImpl prev = gate.enter(prj); try { @@ -489,12 +557,16 @@ public void removeAll(IgnitePredicate>... filter) { @Override public void loadAll(Set keys, boolean replaceExistingValues, CompletionListener completionLsnr) { + onAccess(); + // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public void put(K key, V val) { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -512,6 +584,8 @@ public void removeAll(IgnitePredicate>... filter) { /** {@inheritDoc} */ @Override public V getAndPut(K key, V val) { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -529,6 +603,8 @@ public void removeAll(IgnitePredicate>... filter) { /** {@inheritDoc} */ @Override public void putAll(Map map) { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -546,6 +622,8 @@ public void removeAll(IgnitePredicate>... filter) { /** {@inheritDoc} */ @Override public boolean putIfAbsent(K key, V val) { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -563,6 +641,8 @@ public void removeAll(IgnitePredicate>... filter) { /** {@inheritDoc} */ @Override public boolean remove(K key) { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -580,6 +660,8 @@ public void removeAll(IgnitePredicate>... filter) { /** {@inheritDoc} */ @Override public boolean remove(K key, V oldVal) { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -597,6 +679,8 @@ public void removeAll(IgnitePredicate>... filter) { /** {@inheritDoc} */ @Override public V getAndRemove(K key) { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -614,6 +698,8 @@ public void removeAll(IgnitePredicate>... filter) { /** {@inheritDoc} */ @Override public boolean replace(K key, V oldVal, V newVal) { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -631,6 +717,8 @@ public void removeAll(IgnitePredicate>... filter) { /** {@inheritDoc} */ @Override public boolean replace(K key, V val) { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -648,6 +736,8 @@ public void removeAll(IgnitePredicate>... filter) { /** {@inheritDoc} */ @Override public V getAndReplace(K key, V val) { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -665,6 +755,8 @@ public void removeAll(IgnitePredicate>... filter) { /** {@inheritDoc} */ @Override public void removeAll(Set keys) { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -684,6 +776,8 @@ public void removeAll(IgnitePredicate>... filter) { * @param keys Keys to remove. */ public void removeAll(Collection keys) { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -701,6 +795,8 @@ public void removeAll(Collection keys) { /** {@inheritDoc} */ @Override public void removeAll() { + onAccess(); + // TODO IGNITE-1. GridCacheProjectionImpl prev = gate.enter(prj); @@ -717,6 +813,8 @@ public void removeAll(Collection keys) { /** {@inheritDoc} */ @Override public void clear() { + onAccess(); + // TODO IGNITE-1. GridCacheProjectionImpl prev = gate.enter(prj); @@ -734,6 +832,8 @@ public void removeAll(Collection keys) { /** {@inheritDoc} */ @Override public T invoke(K key, EntryProcessor entryProcessor, Object... args) throws EntryProcessorException { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -773,6 +873,8 @@ public void removeAll(Collection keys) { @Override public Map> invokeAll(Set keys, EntryProcessor entryProcessor, Object... args) { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -792,6 +894,8 @@ public void removeAll(Collection keys) { @Override public Map> invokeAll( Map> map, Object... args) { + onAccess(); + try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -809,11 +913,15 @@ public void removeAll(Collection keys) { /** {@inheritDoc} */ @Override public String getName() { + onAccess(); + return delegate.name(); } /** {@inheritDoc} */ @Override public CacheManager getCacheManager() { + onAccess(); + // TODO IGNITE-45 (Support start/close/destroy cache correctly) IgniteCachingProvider provider = (IgniteCachingProvider)Caching.getCachingProvider( IgniteCachingProvider.class.getName(), @@ -827,12 +935,16 @@ public void removeAll(Collection keys) { /** {@inheritDoc} */ @Override public void close() { + onAccess(); + // TODO IGNITE-45 (Support start/close/destroy cache correctly) getCacheManager().destroyCache(getName()); } /** {@inheritDoc} */ @Override public boolean isClosed() { + onAccess(); + // TODO IGNITE-45 (Support start/close/destroy cache correctly) return getCacheManager() == null; } @@ -840,6 +952,8 @@ public void removeAll(Collection keys) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public T unwrap(Class clazz) { + onAccess(); + if (clazz.equals(IgniteCache.class)) return (T)this; @@ -848,22 +962,28 @@ public void removeAll(Collection keys) { /** {@inheritDoc} */ @Override public void registerCacheEntryListener(CacheEntryListenerConfiguration cacheEntryLsnrConfiguration) { + onAccess(); + // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public void deregisterCacheEntryListener(CacheEntryListenerConfiguration cacheEntryLsnrConfiguration) { + onAccess(); + // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public Iterator> iterator() { + onAccess(); + GridCacheProjectionImpl prev = gate.enter(prj); try { - return new IgniteCacheIterator(); + return new IgniteCacheIterator(delegate.queries().createScanQuery(null).execute(), queryStorage); } finally { gate.leave(prev); @@ -872,42 +992,56 @@ public void removeAll(Collection keys) { /** {@inheritDoc} */ @Override public QueryCursor> query(QueryPredicate filter) { + onAccess(); + // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public QueryCursor query(QueryReducer, R> rmtRdc, QueryPredicate filter) { + onAccess(); + // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public QueryCursor> queryFields(QuerySqlPredicate filter) { + onAccess(); + // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public QueryCursor queryFields(QueryReducer, R> rmtRdc, QuerySqlPredicate filter) { + onAccess(); + // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public QueryCursor> localQuery(QueryPredicate filter) { + onAccess(); + // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public QueryCursor> localQueryFields(QuerySqlPredicate filter) { + onAccess(); + // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public IgniteCache enableAsync() { + onAccess(); + if (isAsync()) return this; @@ -917,6 +1051,8 @@ public void removeAll(Collection keys) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public IgniteCache keepPortable() { + onAccess(); + GridCacheProjectionImpl prev = gate.enter(prj); try { @@ -942,6 +1078,8 @@ public void removeAll(Collection keys) { /** {@inheritDoc} */ @Override public IgniteCache flagsOn(@Nullable GridCacheFlag... flags) { + onAccess(); + GridCacheProjectionImpl prev = gate.enter(prj); try { @@ -987,6 +1125,8 @@ private CacheException cacheException(IgniteCheckedException e) { /** {@inheritDoc} */ @Override public void writeExternal(ObjectOutput out) throws IOException { + onAccess(); + out.writeObject(ctx); out.writeObject(delegate); @@ -997,6 +1137,8 @@ private CacheException cacheException(IgniteCheckedException e) { /** {@inheritDoc} */ @SuppressWarnings({"unchecked"}) @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + onAccess(); + ctx = (GridCacheContext)in.readObject(); delegate = (GridCacheProjectionEx)in.readObject(); @@ -1006,6 +1148,17 @@ private CacheException cacheException(IgniteCheckedException e) { gate = ctx.gate(); } + /** + * Checks if set was removed and handles iterators weak reference queue. + */ + private void onAccess() { + try { + queryStorage.onAccess(); + } catch (IgniteCheckedException e) { + throw cacheException(e); + } + } + /** {@inheritDoc} */ @Override public String toString() { return S.toString(IgniteCacheProxy.class, this); @@ -1085,90 +1238,57 @@ private LoadCacheClosure(String cacheName, IgniteBiPredicate p, Object[] a * Iterator over the cache. */ private class IgniteCacheIterator implements Iterator> { - /** Cache query future for all entries in distributed ignite cache. */ - private GridCacheQueryFuture> fut; - - /** Current element. */ - private Map.Entry curEntry; - /** Next element. */ - private Map.Entry nextEntry; + /** Iterator over the cache*/ + IgniteQueryFutureStorage.Iterator> iter; - /** Init first time. */ - private boolean firstTime = true; + IgniteCacheIterator(GridCacheQueryFuture> fut, IgniteQueryFutureStorage storage) { + iter = storage.iterator(fut); + } /** {@inheritDoc} */ @Override public boolean hasNext() { - initFirstTime(); - - return nextEntry != null; + try { + return iter.onHasNext(); + } catch (IgniteCheckedException e) { + throw cacheException(e); + } } /** {@inheritDoc} */ @Override public Entry next() { - initFirstTime(); - - curEntry = nextEntry; - - if (curEntry == null) - throw new NoSuchElementException(); - try { - nextEntry = fut.next(); + final Map.Entry cur = iter.onNext(); + return new Cache.Entry() { + @Override public K getKey() { + return cur.getKey(); + } + + @Override public V getValue() { + return cur.getValue(); + } + + @Override public T unwrap(Class clazz) { + throw new IllegalArgumentException(); + } + }; } catch (IgniteCheckedException e) { - curEntry = null; - throw cacheException(e); } - return new Cache.Entry() { - @Override public K getKey() { - return curEntry.getKey(); - } - - @Override public V getValue() { - return curEntry.getValue(); - } - @Override public T unwrap(Class clazz) { - throw new IllegalArgumentException(); - } - }; } /** {@inheritDoc} */ @Override public void remove() { - if (curEntry == null) - throw new IllegalStateException(); - + Map.Entry curEntry = iter.itemToRemove(); try { delegate.removex(curEntry.getKey()); } catch (IgniteCheckedException e) { throw cacheException(e); } - - curEntry = null; - } - - /** - * Initialize fields at first call - */ - private void initFirstTime() { - if (!firstTime) { - return; - } - - firstTime = false; - fut = delegate.queries().createScanQuery(null).execute(); - - try { - nextEntry = fut.next(); - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } } } } diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/IgniteQueryFutureStorage.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/IgniteQueryFutureStorage.java new file mode 100644 index 0000000000000..69ef38ba62d51 --- /dev/null +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/IgniteQueryFutureStorage.java @@ -0,0 +1,192 @@ +package org.gridgain.grid.kernal.processors.cache.datastructures; + +import org.apache.ignite.*; +import org.gridgain.grid.cache.query.*; +import org.gridgain.grid.kernal.processors.cache.*; +import org.jdk8.backport.*; + +import java.lang.ref.*; +import java.util.*; + +/** + * Storage for GridCacheQueryFuture. + */ +public class IgniteQueryFutureStorage { + /** Iterators weak references queue. */ + private final ReferenceQueue> refQueue = new ReferenceQueue<>(); + + /** Iterators futures. */ + private final Map>, GridCacheQueryFuture> futs = new ConcurrentHashMap8<>(); + + /** Logger. */ + private final IgniteLogger log; + + /** + * @param ctx Cache context. + */ + public IgniteQueryFutureStorage(GridCacheContext ctx) { + log = ctx.logger(GridCacheSetImpl.class); + } + + /** + * Iterator over the cache. + * @param fut Query to iterate + * @return iterator + */ + public Iterator iterator(GridCacheQueryFuture fut) { + Iterator it = new Iterator<>(fut); + + futs.put(it.weakReference(), fut); + + return it; + } + + /** + * Closes unreachable iterators. + */ + private void checkWeakQueue() throws IgniteCheckedException { + for (Reference> itRef = refQueue.poll(); itRef != null; itRef = refQueue.poll()) { + WeakReference> weakRef = (WeakReference>) itRef; + + GridCacheQueryFuture fut = futs.remove(weakRef); + + if (fut != null) + fut.cancel(); + + } + } + + /** + * Checks if set was removed and handles iterators weak reference queue. + */ + public void onAccess() throws IgniteCheckedException { + checkWeakQueue(); + } + + /** + * Cancel all cache queries + * @throws IgniteCheckedException + */ + protected void clearQueries() throws IgniteCheckedException { + for (GridCacheQueryFuture fut : futs.values()) { + try { + fut.cancel(); + } + catch (IgniteCheckedException e) { + log.error("Failed to close iterator.", e); + } + + } + futs.clear(); + } + + /** + * Iterator over the cache + */ + public class Iterator { + /** Query future. */ + private final GridCacheQueryFuture fut; + + /** Weak reference. */ + private final WeakReference> weakRef; + + /** Init flag. */ + private boolean init; + + /** Next item. */ + private T next; + + /** Current item. */ + private T cur; + + /** + * @param fut GridCacheQueryFuture to iterate + */ + Iterator(GridCacheQueryFuture fut) { + this.fut = fut; + this.weakRef = new WeakReference>(this, refQueue); + } + + + /** + * @throws IgniteCheckedException If failed. + */ + private void init() throws IgniteCheckedException { + if (!init) { + next = fut.next(); + + init = true; + } + } + + /** + * @return Iterator weak reference. + */ + WeakReference> weakReference() { + return weakRef; + } + + /** + * Clears weak reference. + */ + private void clearWeakReference() { + weakRef.clear(); + + futs.remove(weakRef); + } + + /** + * The same as Iterator.next() + */ + public T onNext() throws IgniteCheckedException { + init(); + + if (next == null) { + clearWeakReference(); + + throw new NoSuchElementException(); + } + + cur = next; + + Map.Entry e = (Map.Entry) fut.next(); + + next = e != null ? (T) e.getKey() : null; + + if (next == null) + clearWeakReference(); + + return cur; + } + + /** + * The same as Iterator.hasNext() + */ + public boolean onHasNext() throws IgniteCheckedException { + init(); + + boolean hasNext = next != null; + + if (!hasNext) + clearWeakReference(); + + return hasNext; + } + + /** + * @return current item to remove + * @throws IllegalStateException if the {@code onNext} method has not + * yet been called, or the {@code itemToRemove} method has already + * been called after the last call to the {@code onNext} + * method + */ + public T itemToRemove() { + if (cur == null) + throw new IllegalStateException(); + T res = cur; + cur = null; + return res; + } + } + +} From 68f704edf87e34754ded1ea5a1bf7faa32c98c61 Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Thu, 15 Jan 2015 15:34:46 +0400 Subject: [PATCH 09/36] #IGNITE-53: wip --- .../cache/datastructures/IgniteQueryFutureStorage.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/IgniteQueryFutureStorage.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/IgniteQueryFutureStorage.java index 69ef38ba62d51..4a34036c58e1c 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/IgniteQueryFutureStorage.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/IgniteQueryFutureStorage.java @@ -148,10 +148,7 @@ public T onNext() throws IgniteCheckedException { } cur = next; - - Map.Entry e = (Map.Entry) fut.next(); - - next = e != null ? (T) e.getKey() : null; + next = fut.next(); if (next == null) clearWeakReference(); From d6372ea5a5f4e0bd71cd67f4925490682478fdf1 Mon Sep 17 00:00:00 2001 From: sboikov Date: Thu, 15 Jan 2015 15:57:28 +0300 Subject: [PATCH 10/36] # ignite-53 review --- .../internal/processors/cache/IgniteCacheProxy.java | 4 ++-- .../datastructures/IgniteQueryFutureStorage.java | 3 ++- .../cache/GridCacheAbstractFullApiSelfTest.java | 13 ++++++++++--- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index c41f44eeb41d6..cda62b266a2bb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -1154,7 +1154,8 @@ private CacheException cacheException(IgniteCheckedException e) { private void onAccess() { try { queryStorage.onAccess(); - } catch (IgniteCheckedException e) { + } + catch (IgniteCheckedException e) { throw cacheException(e); } } @@ -1238,7 +1239,6 @@ private LoadCacheClosure(String cacheName, IgniteBiPredicate p, Object[] a * Iterator over the cache. */ private class IgniteCacheIterator implements Iterator> { - /** Iterator over the cache*/ IgniteQueryFutureStorage.Iterator> iter; diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/IgniteQueryFutureStorage.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/IgniteQueryFutureStorage.java index 4a34036c58e1c..d24045f77ee46 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/IgniteQueryFutureStorage.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/IgniteQueryFutureStorage.java @@ -75,8 +75,8 @@ protected void clearQueries() throws IgniteCheckedException { catch (IgniteCheckedException e) { log.error("Failed to close iterator.", e); } - } + futs.clear(); } @@ -104,6 +104,7 @@ public class Iterator { */ Iterator(GridCacheQueryFuture fut) { this.fut = fut; + this.weakRef = new WeakReference>(this, refQueue); } diff --git a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java index 62c364e7744cf..03a60cbaea1db 100644 --- a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java +++ b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java @@ -5257,10 +5257,12 @@ public void testIgniteCacheIterator() throws Exception { IgniteCache cache = jcache(0); final int cacheSz = 100; + Map entries = new HashMap(); for (int i = 0; i < cacheSz; ++i) { cache.put(Integer.toString(i), i); + entries.put(Integer.toString(i), i); } @@ -5291,20 +5293,24 @@ private void checkIteratorHasNext() { * @param entries Expected entries in the cache. */ private void checkIteratorRemove(IgniteCache cache, Map entries) { - //Check that we can remove element. + // Check that we can remove element. String rmvKey = Integer.toString(5); + removeCacheIterator(cache, rmvKey); + entries.remove(rmvKey); + assertFalse(cache.containsKey(rmvKey)); checkIteratorCache(entries); - //check that we cannot call Iterator.remove() without next(). + // Check that we cannot call Iterator.remove() without next(). Iterator> iter = jcache(0).iterator(); assertTrue(iter.hasNext()); iter.next(); + iter.remove(); try { @@ -5322,6 +5328,7 @@ private void checkIteratorRemove(IgniteCache cache, Map cache, String key) { Iterator> iter = cache.iterator(); + int delCnt = 0; while (iter.hasNext()) { @@ -5329,9 +5336,9 @@ private void removeCacheIterator(IgniteCache cache, String key) if (cur.getKey().equals(key)) { iter.remove(); + delCnt++; } - } assertEquals(1, delCnt); From 224f7e2fcac84302e31478de8faaaecbc290c250 Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Thu, 15 Jan 2015 19:16:57 +0400 Subject: [PATCH 11/36] #IGNITE-53: Refactoring IgniteQueryStorage --- .../processors/cache/IgniteCacheProxy.java | 69 ++---- .../datastructures/GridCacheSetImpl.java | 168 ++----------- .../IgniteQueryAbstractStorage.java | 233 ++++++++++++++++++ .../IgniteQueryFutureStorage.java | 190 -------------- 4 files changed, 276 insertions(+), 384 deletions(-) create mode 100644 modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/IgniteQueryAbstractStorage.java delete mode 100644 modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/IgniteQueryFutureStorage.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index cda62b266a2bb..b0ae9138048a2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -24,7 +24,6 @@ import org.apache.ignite.lang.*; import org.apache.ignite.resources.*; import org.gridgain.grid.cache.*; -import org.gridgain.grid.cache.query.*; import org.gridgain.grid.kernal.*; import org.gridgain.grid.kernal.processors.cache.*; import org.gridgain.grid.kernal.processors.cache.datastructures.*; @@ -63,8 +62,8 @@ public class IgniteCacheProxy extends IgniteAsyncSupportAdapter implements /** Projection. */ private GridCacheProjectionImpl prj; - /** Query future storage */ - private final IgniteQueryFutureStorage queryStorage; + /** Query storage */ + private final IgniteQueryStorage queryStorage; /** * @param ctx Context. @@ -85,7 +84,7 @@ public IgniteCacheProxy(GridCacheContext ctx, this.delegate = delegate; this.prj = prj; - this.queryStorage = new IgniteQueryFutureStorage(ctx); + this.queryStorage = new IgniteQueryStorage(ctx); gate = ctx.gate(); } @@ -983,7 +982,7 @@ public void removeAll(Collection keys) { GridCacheProjectionImpl prev = gate.enter(prj); try { - return new IgniteCacheIterator(delegate.queries().createScanQuery(null).execute(), queryStorage); + return queryStorage.iterator(delegate.queries().createScanQuery(null).execute()); } finally { gate.leave(prev); @@ -1236,55 +1235,37 @@ private LoadCacheClosure(String cacheName, IgniteBiPredicate p, Object[] a } /** - * Iterator over the cache. + * Queries' storage */ - private class IgniteCacheIterator implements Iterator> { - /** Iterator over the cache*/ - IgniteQueryFutureStorage.Iterator> iter; - - IgniteCacheIterator(GridCacheQueryFuture> fut, IgniteQueryFutureStorage storage) { - iter = storage.iterator(fut); - } - - /** {@inheritDoc} */ - @Override public boolean hasNext() { - try { - return iter.onHasNext(); - } catch (IgniteCheckedException e) { - throw cacheException(e); - } + private class IgniteQueryStorage extends IgniteQueryAbstractStorage, Map.Entry> { + /** + * @param ctx Cache context. + */ + public IgniteQueryStorage(GridCacheContext ctx) { + super(ctx); } /** {@inheritDoc} */ - @Override public Entry next() { - try { - final Map.Entry cur = iter.onNext(); - return new Cache.Entry() { - @Override public K getKey() { - return cur.getKey(); - } - - @Override public V getValue() { - return cur.getValue(); - } - - @Override public T unwrap(Class clazz) { - throw new IllegalArgumentException(); - } - }; - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } + @Override protected Cache.Entry convert(final Map.Entry v) { + return new Cache.Entry() { + @Override public K getKey() { + return v.getKey(); + } + @Override public V getValue() { + return v.getValue(); + } + @Override public T unwrap(Class clazz) { + throw new IllegalArgumentException(); + } + }; } /** {@inheritDoc} */ - @Override public void remove() { - Map.Entry curEntry = iter.itemToRemove(); + @Override protected void remove(Entry item) { try { - delegate.removex(curEntry.getKey()); + delegate.removex(item.getKey()); } catch (IgniteCheckedException e) { throw cacheException(e); diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/GridCacheSetImpl.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/GridCacheSetImpl.java index 9519ad8bf3dc9..23eca5290266e 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/GridCacheSetImpl.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/GridCacheSetImpl.java @@ -20,7 +20,6 @@ import org.apache.ignite.*; import org.apache.ignite.cluster.*; import org.apache.ignite.lang.*; -import org.gridgain.grid.*; import org.gridgain.grid.cache.*; import org.gridgain.grid.cache.affinity.*; import org.gridgain.grid.cache.datastructures.*; @@ -31,11 +30,9 @@ import org.gridgain.grid.util.lang.*; import org.gridgain.grid.util.typedef.*; import org.gridgain.grid.util.typedef.internal.*; -import org.jdk8.backport.*; import org.jetbrains.annotations.*; import java.io.*; -import java.lang.ref.*; import java.util.*; import java.util.concurrent.*; @@ -54,9 +51,6 @@ public class GridCacheSetImpl extends AbstractCollection implements GridCa /** Cache. */ private final GridCache cache; - /** Logger. */ - private final IgniteLogger log; - /** Set name. */ private final String name; @@ -72,11 +66,8 @@ public class GridCacheSetImpl extends AbstractCollection implements GridCa /** Removed flag. */ private volatile boolean rmvd; - /** Iterators weak references queue. */ - private final ReferenceQueue> itRefQueue = new ReferenceQueue<>(); - - /** Iterators futures. */ - private final Map>, GridCacheQueryFuture> itFuts = new ConcurrentHashMap8<>(); + /** Query storage */ + private final IgniteQueryStorage queryStorage; /** * @param ctx Cache context. @@ -92,9 +83,9 @@ public GridCacheSetImpl(GridCacheContext ctx, String name, GridCacheSetHeader hd cache = ctx.cache(); - log = ctx.logger(GridCacheSetImpl.class); - hdrPart = ctx.affinity().partition(new GridCacheSetHeaderKey(name)); + + queryStorage = new IgniteQueryStorage(ctx); } /** {@inheritDoc} */ @@ -349,16 +340,10 @@ private GridCloseableIterator iterator0() { qry.projection(ctx.grid().forNodes(nodes)); - GridCacheQueryFuture fut = qry.execute(); - - SetIterator it = new SetIterator<>(fut); - - itFuts.put(it.weakReference(), fut); + IgniteQueryAbstractStorage.IgniteIterator it = queryStorage.iterator(qry.execute()); if (rmvd) { - itFuts.remove(it.weakReference()); - - it.close(); + queryStorage.removeIterator(it); checkRemoved(); } @@ -444,18 +429,8 @@ void removed(boolean rmvd) { this.rmvd = rmvd; - if (rmvd) { - for (GridCacheQueryFuture fut : itFuts.values()) { - try { - fut.cancel(); - } - catch (IgniteCheckedException e) { - log.error("Failed to close iterator.", e); - } - } - - itFuts.clear(); - } + if (rmvd) + queryStorage.clearQueries(); } /** @@ -466,30 +441,11 @@ private void checkRemoved() { throw new GridCacheDataStructureRemovedRuntimeException("Set has been removed from cache: " + this); } - /** - * Closes unreachable iterators. - */ - private void checkWeakQueue() { - for (Reference> itRef = itRefQueue.poll(); itRef != null; itRef = itRefQueue.poll()) { - try { - WeakReference> weakRef = (WeakReference>)itRef; - - GridCacheQueryFuture fut = itFuts.remove(weakRef); - - if (fut != null) - fut.cancel(); - } - catch (IgniteCheckedException e) { - log.error("Failed to close iterator.", e); - } - } - } - /** * Checks if set was removed and handles iterators weak reference queue. */ private void onAccess() { - checkWeakQueue(); + queryStorage.checkWeakQueue(); checkRemoved(); } @@ -522,112 +478,24 @@ private GridCacheSetItemKey itemKey(Object item) { } /** - * + * Queries' storage. */ - private class SetIterator extends GridCloseableIteratorAdapter { - /** */ - private static final long serialVersionUID = -1460570789166994846L; - - /** Query future. */ - private final GridCacheQueryFuture fut; - - /** Init flag. */ - private boolean init; - - /** Next item. */ - private T next; - - /** Current item. */ - private T cur; - - /** Weak reference. */ - private final WeakReference> weakRef; - + private class IgniteQueryStorage extends IgniteQueryAbstractStorage> { /** - * @param fut Query future. + * @param ctx Cache context. */ - private SetIterator(GridCacheQueryFuture fut) { - this.fut = fut; - - weakRef = new WeakReference>(this, itRefQueue); - } - - /** {@inheritDoc} */ - @Override protected T onNext() throws IgniteCheckedException { - init(); - - if (next == null) { - clearWeakReference(); - - throw new NoSuchElementException(); - } - - cur = next; - - Map.Entry e = (Map.Entry)fut.next(); - - next = e != null ? (T)e.getKey() : null; - - if (next == null) - clearWeakReference(); - - return cur; + public IgniteQueryStorage(GridCacheContext ctx) { + super(ctx); } /** {@inheritDoc} */ - @Override protected boolean onHasNext() throws IgniteCheckedException { - init(); - - boolean hasNext = next != null; - - if (!hasNext) - clearWeakReference(); - - return hasNext; - } - - /** {@inheritDoc} */ - @Override protected void onClose() throws IgniteCheckedException { - fut.cancel(); - - clearWeakReference(); + @Override protected T convert(Map.Entry v) { + return v != null ? (T) v.getKey() : null; } /** {@inheritDoc} */ - @Override protected void onRemove() throws IgniteCheckedException { - if (cur == null) - throw new NoSuchElementException(); - - GridCacheSetImpl.this.remove(cur); - } - - /** - * @throws IgniteCheckedException If failed. - */ - private void init() throws IgniteCheckedException { - if (!init) { - Map.Entry e = (Map.Entry)fut.next(); - - next = e != null ? (T)e.getKey() : null; - - init = true; - } - } - - /** - * @return Iterator weak reference. - */ - WeakReference> weakReference() { - return weakRef; - } - - /** - * Clears weak reference. - */ - private void clearWeakReference() { - weakRef.clear(); // Do not need to enqueue. - - itFuts.remove(weakRef); + @Override protected void remove(T item) { + GridCacheSetImpl.this.remove(item); } } diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/IgniteQueryAbstractStorage.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/IgniteQueryAbstractStorage.java new file mode 100644 index 0000000000000..443fb86207bcb --- /dev/null +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/IgniteQueryAbstractStorage.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gridgain.grid.kernal.processors.cache.datastructures; + +import org.apache.ignite.*; +import org.gridgain.grid.cache.query.*; +import org.gridgain.grid.kernal.processors.cache.*; +import org.gridgain.grid.util.*; +import org.jdk8.backport.*; + +import java.lang.ref.*; +import java.util.*; + +/** + * Storage for GridCacheQueryFuture. + * @param Type for iterator. + * @param Type for cache query future. + */ +public abstract class IgniteQueryAbstractStorage { + /** Iterators weak references queue. */ + private final ReferenceQueue refQueue = new ReferenceQueue<>(); + + /** Iterators futures. */ + private final Map, GridCacheQueryFuture> futs = new ConcurrentHashMap8<>(); + + /** Logger. */ + private final IgniteLogger log; + + /** + * @param ctx Cache context. + */ + public IgniteQueryAbstractStorage(GridCacheContext ctx) { + log = ctx.logger(IgniteQueryAbstractStorage.class); + } + + /** + * Iterator over the cache. + * @param fut Query to iterate + * @return iterator + */ + public IgniteIterator iterator(GridCacheQueryFuture fut) { + IgniteIterator it = new IgniteIterator(fut); + + futs.put(it.weakReference(), fut); + + return it; + } + + public void removeIterator(IgniteIterator it) throws IgniteCheckedException { + futs.remove(it.weakReference()); + + it.close(); + } + + /** + * Closes unreachable iterators. + */ + public void checkWeakQueue() { + for (Reference itRef = refQueue.poll(); itRef != null; itRef = refQueue.poll()) { + try { + WeakReference weakRef = (WeakReference)itRef; + + GridCacheQueryFuture fut = futs.remove(weakRef); + + if (fut != null) + fut.cancel(); + } + catch (IgniteCheckedException e) { + log.error("Failed to close iterator.", e); + } + } + } + + /** + * Checks if set was removed and handles iterators weak reference queue. + */ + public void onAccess() throws IgniteCheckedException { + checkWeakQueue(); + } + + /** + * Cancel all cache queries + */ + protected void clearQueries(){ + for (GridCacheQueryFuture fut : futs.values()) { + try { + fut.cancel(); + } + catch (IgniteCheckedException e) { + log.error("Failed to close iterator.", e); + } + } + + futs.clear(); + } + + /** + * Convert class V to class T. + * @param v Item to convert. + * @return Converted item. + */ + protected abstract T convert(V v); + + /** + * Remove item from the cache. + * @param item Item to remove. + */ + protected abstract void remove(T item); + + /** + * Iterator over the cache. + */ + public class IgniteIterator extends GridCloseableIteratorAdapter { + /** Query future. */ + private final GridCacheQueryFuture fut; + + /** Weak reference. */ + private final WeakReference weakRef; + + /** Init flag. */ + private boolean init; + + /** Next item. */ + private T next; + + /** Current item. */ + private T cur; + + /** + * @param fut GridCacheQueryFuture to iterate + */ + IgniteIterator(GridCacheQueryFuture fut) { + this.fut = fut; + + this.weakRef = new WeakReference(this, refQueue); + } + + /** + * @return Iterator weak reference. + */ + public WeakReference weakReference() { + return weakRef; + } + + /** {@inheritDoc} */ + @Override public T onNext() throws IgniteCheckedException { + init(); + + if (next == null) { + clearWeakReference(); + + throw new NoSuchElementException(); + } + + cur = next; + + V futNext = fut.next(); + + if (futNext == null) + clearWeakReference(); + + next = futNext != null ? convert(futNext) : null; + + return cur; + } + + /** {@inheritDoc} */ + @Override public boolean onHasNext() throws IgniteCheckedException { + init(); + + boolean hasNext = next != null; + + if (!hasNext) + clearWeakReference(); + + return hasNext; + } + + /** {@inheritDoc} */ + @Override protected void onClose() throws IgniteCheckedException { + fut.cancel(); + + clearWeakReference(); + } + + /** {@inheritDoc} */ + @Override protected void onRemove() throws IgniteCheckedException { + if (cur == null) + throw new IllegalStateException(); + + IgniteQueryAbstractStorage.this.remove(cur); + + cur = null; + } + + /** + * Clears weak reference. + */ + private void clearWeakReference() { + weakRef.clear(); + + futs.remove(weakRef); + } + + /** + * @throws IgniteCheckedException If failed. + */ + private void init() throws IgniteCheckedException { + if (!init) { + V futNext = fut.next(); + + next = futNext != null ? convert(futNext) : null; + + init = true; + } + } + } +} diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/IgniteQueryFutureStorage.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/IgniteQueryFutureStorage.java deleted file mode 100644 index d24045f77ee46..0000000000000 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/IgniteQueryFutureStorage.java +++ /dev/null @@ -1,190 +0,0 @@ -package org.gridgain.grid.kernal.processors.cache.datastructures; - -import org.apache.ignite.*; -import org.gridgain.grid.cache.query.*; -import org.gridgain.grid.kernal.processors.cache.*; -import org.jdk8.backport.*; - -import java.lang.ref.*; -import java.util.*; - -/** - * Storage for GridCacheQueryFuture. - */ -public class IgniteQueryFutureStorage { - /** Iterators weak references queue. */ - private final ReferenceQueue> refQueue = new ReferenceQueue<>(); - - /** Iterators futures. */ - private final Map>, GridCacheQueryFuture> futs = new ConcurrentHashMap8<>(); - - /** Logger. */ - private final IgniteLogger log; - - /** - * @param ctx Cache context. - */ - public IgniteQueryFutureStorage(GridCacheContext ctx) { - log = ctx.logger(GridCacheSetImpl.class); - } - - /** - * Iterator over the cache. - * @param fut Query to iterate - * @return iterator - */ - public Iterator iterator(GridCacheQueryFuture fut) { - Iterator it = new Iterator<>(fut); - - futs.put(it.weakReference(), fut); - - return it; - } - - /** - * Closes unreachable iterators. - */ - private void checkWeakQueue() throws IgniteCheckedException { - for (Reference> itRef = refQueue.poll(); itRef != null; itRef = refQueue.poll()) { - WeakReference> weakRef = (WeakReference>) itRef; - - GridCacheQueryFuture fut = futs.remove(weakRef); - - if (fut != null) - fut.cancel(); - - } - } - - /** - * Checks if set was removed and handles iterators weak reference queue. - */ - public void onAccess() throws IgniteCheckedException { - checkWeakQueue(); - } - - /** - * Cancel all cache queries - * @throws IgniteCheckedException - */ - protected void clearQueries() throws IgniteCheckedException { - for (GridCacheQueryFuture fut : futs.values()) { - try { - fut.cancel(); - } - catch (IgniteCheckedException e) { - log.error("Failed to close iterator.", e); - } - } - - futs.clear(); - } - - /** - * Iterator over the cache - */ - public class Iterator { - /** Query future. */ - private final GridCacheQueryFuture fut; - - /** Weak reference. */ - private final WeakReference> weakRef; - - /** Init flag. */ - private boolean init; - - /** Next item. */ - private T next; - - /** Current item. */ - private T cur; - - /** - * @param fut GridCacheQueryFuture to iterate - */ - Iterator(GridCacheQueryFuture fut) { - this.fut = fut; - - this.weakRef = new WeakReference>(this, refQueue); - } - - - /** - * @throws IgniteCheckedException If failed. - */ - private void init() throws IgniteCheckedException { - if (!init) { - next = fut.next(); - - init = true; - } - } - - /** - * @return Iterator weak reference. - */ - WeakReference> weakReference() { - return weakRef; - } - - /** - * Clears weak reference. - */ - private void clearWeakReference() { - weakRef.clear(); - - futs.remove(weakRef); - } - - /** - * The same as Iterator.next() - */ - public T onNext() throws IgniteCheckedException { - init(); - - if (next == null) { - clearWeakReference(); - - throw new NoSuchElementException(); - } - - cur = next; - next = fut.next(); - - if (next == null) - clearWeakReference(); - - return cur; - } - - /** - * The same as Iterator.hasNext() - */ - public boolean onHasNext() throws IgniteCheckedException { - init(); - - boolean hasNext = next != null; - - if (!hasNext) - clearWeakReference(); - - return hasNext; - } - - /** - * @return current item to remove - * @throws IllegalStateException if the {@code onNext} method has not - * yet been called, or the {@code itemToRemove} method has already - * been called after the last call to the {@code onNext} - * method - */ - public T itemToRemove() { - if (cur == null) - throw new IllegalStateException(); - T res = cur; - cur = null; - return res; - } - } - -} From c927f689acf599665939eeefa11dd7d7154e52f7 Mon Sep 17 00:00:00 2001 From: sboikov Date: Fri, 16 Jan 2015 10:23:51 +0300 Subject: [PATCH 12/36] # ignite-53 review --- .../processors/cache/IgniteCacheProxy.java | 206 ++---------------- .../processors/cache/CacheEntryImpl.java | 64 ++++++ ...ava => CacheWeakQueryIteratorsHolder.java} | 75 ++++--- .../datastructures/GridCacheSetImpl.java | 44 ++-- .../GridCacheAbstractFullApiSelfTest.java | 23 +- 5 files changed, 143 insertions(+), 269 deletions(-) create mode 100644 modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheEntryImpl.java rename modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/{datastructures/IgniteQueryAbstractStorage.java => CacheWeakQueryIteratorsHolder.java} (72%) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index b0ae9138048a2..9b78fb19d6b9b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -26,7 +26,6 @@ import org.gridgain.grid.cache.*; import org.gridgain.grid.kernal.*; import org.gridgain.grid.kernal.processors.cache.*; -import org.gridgain.grid.kernal.processors.cache.datastructures.*; import org.gridgain.grid.util.tostring.*; import org.gridgain.grid.util.typedef.*; import org.gridgain.grid.util.typedef.internal.*; @@ -62,8 +61,8 @@ public class IgniteCacheProxy extends IgniteAsyncSupportAdapter implements /** Projection. */ private GridCacheProjectionImpl prj; - /** Query storage */ - private final IgniteQueryStorage queryStorage; + /** Iterator holder. */ + private final CacheWeakQueryIteratorsHolder, Map.Entry> itHolder; /** * @param ctx Context. @@ -84,7 +83,15 @@ public IgniteCacheProxy(GridCacheContext ctx, this.delegate = delegate; this.prj = prj; - this.queryStorage = new IgniteQueryStorage(ctx); + this.itHolder = new CacheWeakQueryIteratorsHolder, Map.Entry>(ctx.logger(IgniteCacheProxy.class)) { + @Override protected Entry convert(Map.Entry e) { + return new CacheEntryImpl<>(e.getKey(), e.getValue()); + } + + @Override protected void remove(Entry item) { + IgniteCacheProxy.this.remove(item.getKey()); + } + }; gate = ctx.gate(); } @@ -93,8 +100,6 @@ public IgniteCacheProxy(GridCacheContext ctx, * @return Context. */ public GridCacheContext context() { - onAccess(); - return ctx; } @@ -102,15 +107,11 @@ public GridCacheContext context() { * @return Ignite instance. */ @Override public GridEx ignite() { - onAccess(); - return ctx.grid(); } /** {@inheritDoc} */ @Override public > C getConfiguration(Class clazz) { - onAccess(); - GridCacheConfiguration cfg = ctx.config(); if (!clazz.isAssignableFrom(cfg.getClass())) @@ -121,16 +122,12 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Nullable @Override public Entry randomEntry() { - onAccess(); - // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public IgniteCache withExpiryPolicy(ExpiryPolicy plc) { - onAccess(); - GridCacheProjectionImpl prev = gate.enter(prj); try { @@ -145,8 +142,6 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public void loadCache(@Nullable IgniteBiPredicate p, @Nullable Object... args) { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -168,8 +163,6 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public void localLoadCache(@Nullable IgniteBiPredicate p, @Nullable Object... args) { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -187,8 +180,6 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Nullable @Override public V getAndPutIf(K key, V val, IgnitePredicate> filter) { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -206,8 +197,6 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public boolean putIf(K key, V val, IgnitePredicate> filter) { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -225,8 +214,6 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public V getAndRemoveIf(K key, IgnitePredicate> filter) { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -244,8 +231,6 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public boolean removeIf(K key, IgnitePredicate> filter) { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -263,8 +248,6 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Nullable @Override public V getAndPutIfAbsent(K key, V val) throws CacheException { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -282,32 +265,24 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public void removeAll(IgnitePredicate filter) throws CacheException { - onAccess(); - // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public Lock lock(K key) throws CacheException { - onAccess(); - // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public Lock lockAll(Set keys) throws CacheException { - onAccess(); - // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public boolean isLocked(K key) { - onAccess(); - GridCacheProjectionImpl prev = gate.enter(prj); try { @@ -320,8 +295,6 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public boolean isLockedByThread(K key) { - onAccess(); - GridCacheProjectionImpl prev = gate.enter(prj); try { @@ -334,24 +307,18 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public Iterable> localEntries(CachePeekMode... peekModes) throws CacheException { - onAccess(); - // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public Map localPartition(int part) throws CacheException { - onAccess(); - // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public void localEvict(Collection keys) { - onAccess(); - GridCacheProjectionImpl prev = gate.enter(prj); try { @@ -364,8 +331,6 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Nullable @Override public V localPeek(K key, CachePeekMode... peekModes) { - onAccess(); - // TODO IGNITE-1. if (peekModes.length != 0) throw new UnsupportedOperationException(); @@ -382,8 +347,6 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public void localPromote(Set keys) throws CacheException { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -401,16 +364,12 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public boolean clear(Collection keys) { - onAccess(); - // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public int size(CachePeekMode... peekModes) throws CacheException { - onAccess(); - // TODO IGNITE-1. if (peekModes.length != 0) throw new UnsupportedOperationException(); @@ -427,8 +386,6 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public int localSize(CachePeekMode... peekModes) { - onAccess(); - // TODO IGNITE-1. GridCacheProjectionImpl prev = gate.enter(prj); @@ -442,8 +399,6 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public V get(K key) { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -461,8 +416,6 @@ public GridCacheContext context() { /** {@inheritDoc} */ @Override public Map getAll(Set keys) { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -483,8 +436,6 @@ public GridCacheContext context() { * @return Values map. */ public Map getAll(Collection keys) { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -507,8 +458,6 @@ public Map getAll(Collection keys) { * @return Entry set. */ public Set> entrySetx(IgnitePredicate>... filter) { - onAccess(); - GridCacheProjectionImpl prev = gate.enter(prj); try { @@ -523,8 +472,6 @@ public Set> entrySetx(IgnitePredicate> * @param filter Filter. */ public void removeAll(IgnitePredicate>... filter) { - onAccess(); - GridCacheProjectionImpl prev = gate.enter(prj); try { @@ -540,8 +487,6 @@ public void removeAll(IgnitePredicate>... filter) { /** {@inheritDoc} */ @Override public boolean containsKey(K key) { - onAccess(); - GridCacheProjectionImpl prev = gate.enter(prj); try { @@ -556,16 +501,12 @@ public void removeAll(IgnitePredicate>... filter) { @Override public void loadAll(Set keys, boolean replaceExistingValues, CompletionListener completionLsnr) { - onAccess(); - // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public void put(K key, V val) { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -583,8 +524,6 @@ public void removeAll(IgnitePredicate>... filter) { /** {@inheritDoc} */ @Override public V getAndPut(K key, V val) { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -602,8 +541,6 @@ public void removeAll(IgnitePredicate>... filter) { /** {@inheritDoc} */ @Override public void putAll(Map map) { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -621,8 +558,6 @@ public void removeAll(IgnitePredicate>... filter) { /** {@inheritDoc} */ @Override public boolean putIfAbsent(K key, V val) { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -640,8 +575,6 @@ public void removeAll(IgnitePredicate>... filter) { /** {@inheritDoc} */ @Override public boolean remove(K key) { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -659,8 +592,6 @@ public void removeAll(IgnitePredicate>... filter) { /** {@inheritDoc} */ @Override public boolean remove(K key, V oldVal) { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -678,8 +609,6 @@ public void removeAll(IgnitePredicate>... filter) { /** {@inheritDoc} */ @Override public V getAndRemove(K key) { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -697,8 +626,6 @@ public void removeAll(IgnitePredicate>... filter) { /** {@inheritDoc} */ @Override public boolean replace(K key, V oldVal, V newVal) { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -716,8 +643,6 @@ public void removeAll(IgnitePredicate>... filter) { /** {@inheritDoc} */ @Override public boolean replace(K key, V val) { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -735,8 +660,6 @@ public void removeAll(IgnitePredicate>... filter) { /** {@inheritDoc} */ @Override public V getAndReplace(K key, V val) { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -754,8 +677,6 @@ public void removeAll(IgnitePredicate>... filter) { /** {@inheritDoc} */ @Override public void removeAll(Set keys) { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -775,8 +696,6 @@ public void removeAll(IgnitePredicate>... filter) { * @param keys Keys to remove. */ public void removeAll(Collection keys) { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -794,8 +713,6 @@ public void removeAll(Collection keys) { /** {@inheritDoc} */ @Override public void removeAll() { - onAccess(); - // TODO IGNITE-1. GridCacheProjectionImpl prev = gate.enter(prj); @@ -812,8 +729,6 @@ public void removeAll(Collection keys) { /** {@inheritDoc} */ @Override public void clear() { - onAccess(); - // TODO IGNITE-1. GridCacheProjectionImpl prev = gate.enter(prj); @@ -831,8 +746,6 @@ public void removeAll(Collection keys) { /** {@inheritDoc} */ @Override public T invoke(K key, EntryProcessor entryProcessor, Object... args) throws EntryProcessorException { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -872,8 +785,6 @@ public void removeAll(Collection keys) { @Override public Map> invokeAll(Set keys, EntryProcessor entryProcessor, Object... args) { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -893,8 +804,6 @@ public void removeAll(Collection keys) { @Override public Map> invokeAll( Map> map, Object... args) { - onAccess(); - try { GridCacheProjectionImpl prev = gate.enter(prj); @@ -912,15 +821,11 @@ public void removeAll(Collection keys) { /** {@inheritDoc} */ @Override public String getName() { - onAccess(); - return delegate.name(); } /** {@inheritDoc} */ @Override public CacheManager getCacheManager() { - onAccess(); - // TODO IGNITE-45 (Support start/close/destroy cache correctly) IgniteCachingProvider provider = (IgniteCachingProvider)Caching.getCachingProvider( IgniteCachingProvider.class.getName(), @@ -934,16 +839,12 @@ public void removeAll(Collection keys) { /** {@inheritDoc} */ @Override public void close() { - onAccess(); - // TODO IGNITE-45 (Support start/close/destroy cache correctly) getCacheManager().destroyCache(getName()); } /** {@inheritDoc} */ @Override public boolean isClosed() { - onAccess(); - // TODO IGNITE-45 (Support start/close/destroy cache correctly) return getCacheManager() == null; } @@ -951,8 +852,6 @@ public void removeAll(Collection keys) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public T unwrap(Class clazz) { - onAccess(); - if (clazz.equals(IgniteCache.class)) return (T)this; @@ -961,28 +860,24 @@ public void removeAll(Collection keys) { /** {@inheritDoc} */ @Override public void registerCacheEntryListener(CacheEntryListenerConfiguration cacheEntryLsnrConfiguration) { - onAccess(); - // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public void deregisterCacheEntryListener(CacheEntryListenerConfiguration cacheEntryLsnrConfiguration) { - onAccess(); - // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public Iterator> iterator() { - onAccess(); - GridCacheProjectionImpl prev = gate.enter(prj); try { - return queryStorage.iterator(delegate.queries().createScanQuery(null).execute()); + itHolder.checkWeakQueue(); + + return itHolder.iterator(delegate.queries().createScanQuery(null).execute()); } finally { gate.leave(prev); @@ -991,56 +886,42 @@ public void removeAll(Collection keys) { /** {@inheritDoc} */ @Override public QueryCursor> query(QueryPredicate filter) { - onAccess(); - // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public QueryCursor query(QueryReducer, R> rmtRdc, QueryPredicate filter) { - onAccess(); - // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public QueryCursor> queryFields(QuerySqlPredicate filter) { - onAccess(); - // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public QueryCursor queryFields(QueryReducer, R> rmtRdc, QuerySqlPredicate filter) { - onAccess(); - // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public QueryCursor> localQuery(QueryPredicate filter) { - onAccess(); - // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public QueryCursor> localQueryFields(QuerySqlPredicate filter) { - onAccess(); - // TODO IGNITE-1. throw new UnsupportedOperationException(); } /** {@inheritDoc} */ @Override public IgniteCache enableAsync() { - onAccess(); - if (isAsync()) return this; @@ -1050,8 +931,6 @@ public void removeAll(Collection keys) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public IgniteCache keepPortable() { - onAccess(); - GridCacheProjectionImpl prev = gate.enter(prj); try { @@ -1077,8 +956,6 @@ public void removeAll(Collection keys) { /** {@inheritDoc} */ @Override public IgniteCache flagsOn(@Nullable GridCacheFlag... flags) { - onAccess(); - GridCacheProjectionImpl prev = gate.enter(prj); try { @@ -1124,8 +1001,6 @@ private CacheException cacheException(IgniteCheckedException e) { /** {@inheritDoc} */ @Override public void writeExternal(ObjectOutput out) throws IOException { - onAccess(); - out.writeObject(ctx); out.writeObject(delegate); @@ -1136,8 +1011,6 @@ private CacheException cacheException(IgniteCheckedException e) { /** {@inheritDoc} */ @SuppressWarnings({"unchecked"}) @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { - onAccess(); - ctx = (GridCacheContext)in.readObject(); delegate = (GridCacheProjectionEx)in.readObject(); @@ -1147,18 +1020,6 @@ private CacheException cacheException(IgniteCheckedException e) { gate = ctx.gate(); } - /** - * Checks if set was removed and handles iterators weak reference queue. - */ - private void onAccess() { - try { - queryStorage.onAccess(); - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - /** {@inheritDoc} */ @Override public String toString() { return S.toString(IgniteCacheProxy.class, this); @@ -1233,43 +1094,4 @@ private LoadCacheClosure(String cacheName, IgniteBiPredicate p, Object[] a return S.toString(LoadCacheClosure.class, this); } } - - /** - * Queries' storage - */ - private class IgniteQueryStorage extends IgniteQueryAbstractStorage, Map.Entry> { - /** - * @param ctx Cache context. - */ - public IgniteQueryStorage(GridCacheContext ctx) { - super(ctx); - } - - /** {@inheritDoc} */ - @Override protected Cache.Entry convert(final Map.Entry v) { - return new Cache.Entry() { - @Override public K getKey() { - return v.getKey(); - } - - @Override public V getValue() { - return v.getValue(); - } - - @Override public T unwrap(Class clazz) { - throw new IllegalArgumentException(); - } - }; - } - - /** {@inheritDoc} */ - @Override protected void remove(Entry item) { - try { - delegate.removex(item.getKey()); - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - } } diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheEntryImpl.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheEntryImpl.java new file mode 100644 index 0000000000000..3a7a546f81720 --- /dev/null +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheEntryImpl.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gridgain.grid.kernal.processors.cache; + +import javax.cache.*; + +/** + * + */ +public class CacheEntryImpl implements Cache.Entry { + /** */ + private final K key; + + /** */ + private final V val; + + /** + * @param key Key. + * @param val Value. + */ + public CacheEntryImpl(K key, V val) { + this.key = key; + this.val = val; + } + + /** {@inheritDoc} */ + @Override public K getKey() { + return key; + } + + /** {@inheritDoc} */ + @Override public V getValue() { + return val; + } + + /** {@inheritDoc} */ + @SuppressWarnings("unchecked") + @Override public T unwrap(Class cls) { + if (!cls.equals(getClass())) + throw new IllegalArgumentException("Unwrapping is not supported: " + cls); + + return (T)this; + } + + /** {@inheritDoc} */ + public String toString() { + return "CacheEntry [key=" + key + ", val=" + val + ']'; + } +} diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/IgniteQueryAbstractStorage.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheWeakQueryIteratorsHolder.java similarity index 72% rename from modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/IgniteQueryAbstractStorage.java rename to modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheWeakQueryIteratorsHolder.java index 443fb86207bcb..eaa65be7a8755 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/IgniteQueryAbstractStorage.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheWeakQueryIteratorsHolder.java @@ -15,11 +15,10 @@ * limitations under the License. */ -package org.gridgain.grid.kernal.processors.cache.datastructures; +package org.gridgain.grid.kernal.processors.cache; import org.apache.ignite.*; import org.gridgain.grid.cache.query.*; -import org.gridgain.grid.kernal.processors.cache.*; import org.gridgain.grid.util.*; import org.jdk8.backport.*; @@ -27,25 +26,25 @@ import java.util.*; /** - * Storage for GridCacheQueryFuture. * @param Type for iterator. * @param Type for cache query future. */ -public abstract class IgniteQueryAbstractStorage { +public abstract class CacheWeakQueryIteratorsHolder { /** Iterators weak references queue. */ - private final ReferenceQueue refQueue = new ReferenceQueue<>(); + private final ReferenceQueue refQueue = new ReferenceQueue<>(); /** Iterators futures. */ - private final Map, GridCacheQueryFuture> futs = new ConcurrentHashMap8<>(); + private final Map, GridCacheQueryFuture> futs = + new ConcurrentHashMap8<>(); /** Logger. */ private final IgniteLogger log; /** - * @param ctx Cache context. + * @param log Logger. */ - public IgniteQueryAbstractStorage(GridCacheContext ctx) { - log = ctx.logger(IgniteQueryAbstractStorage.class); + public CacheWeakQueryIteratorsHolder(IgniteLogger log) { + this.log = log; } /** @@ -53,15 +52,20 @@ public IgniteQueryAbstractStorage(GridCacheContext ctx) { * @param fut Query to iterate * @return iterator */ - public IgniteIterator iterator(GridCacheQueryFuture fut) { - IgniteIterator it = new IgniteIterator(fut); + public WeakQueryFutureIterator iterator(GridCacheQueryFuture fut) { + WeakQueryFutureIterator it = new WeakQueryFutureIterator(fut); futs.put(it.weakReference(), fut); return it; } - public void removeIterator(IgniteIterator it) throws IgniteCheckedException { + /** + * @param it Iterator. + * + * @throws IgniteCheckedException If failed. + */ + public void removeIterator(WeakQueryFutureIterator it) throws IgniteCheckedException { futs.remove(it.weakReference()); it.close(); @@ -71,9 +75,9 @@ public void removeIterator(IgniteIterator it) throws IgniteCheckedException { * Closes unreachable iterators. */ public void checkWeakQueue() { - for (Reference itRef = refQueue.poll(); itRef != null; itRef = refQueue.poll()) { + for (Reference itRef = refQueue.poll(); itRef != null; itRef = refQueue.poll()) { try { - WeakReference weakRef = (WeakReference)itRef; + WeakReference weakRef = (WeakReference)itRef; GridCacheQueryFuture fut = futs.remove(weakRef); @@ -86,17 +90,10 @@ public void checkWeakQueue() { } } - /** - * Checks if set was removed and handles iterators weak reference queue. - */ - public void onAccess() throws IgniteCheckedException { - checkWeakQueue(); - } - /** * Cancel all cache queries */ - protected void clearQueries(){ + public void clearQueries(){ for (GridCacheQueryFuture fut : futs.values()) { try { fut.cancel(); @@ -110,27 +107,29 @@ protected void clearQueries(){ } /** - * Convert class V to class T. + * Converts class V to class T. + * * @param v Item to convert. * @return Converted item. */ protected abstract T convert(V v); /** - * Remove item from the cache. + * Removes item. + * * @param item Item to remove. */ protected abstract void remove(T item); /** - * Iterator over the cache. + * Iterator based of {@link GridCacheQueryFuture}. */ - public class IgniteIterator extends GridCloseableIteratorAdapter { + public class WeakQueryFutureIterator extends GridCloseableIteratorAdapter { /** Query future. */ private final GridCacheQueryFuture fut; /** Weak reference. */ - private final WeakReference weakRef; + private final WeakReference weakRef; /** Init flag. */ private boolean init; @@ -142,19 +141,12 @@ public class IgniteIterator extends GridCloseableIteratorAdapter { private T cur; /** - * @param fut GridCacheQueryFuture to iterate + * @param fut GridCacheQueryFuture to iterate. */ - IgniteIterator(GridCacheQueryFuture fut) { + WeakQueryFutureIterator(GridCacheQueryFuture fut) { this.fut = fut; - this.weakRef = new WeakReference(this, refQueue); - } - - /** - * @return Iterator weak reference. - */ - public WeakReference weakReference() { - return weakRef; + this.weakRef = new WeakReference<>(this, refQueue); } /** {@inheritDoc} */ @@ -203,11 +195,18 @@ public WeakReference weakReference() { if (cur == null) throw new IllegalStateException(); - IgniteQueryAbstractStorage.this.remove(cur); + CacheWeakQueryIteratorsHolder.this.remove(cur); cur = null; } + /** + * @return Iterator weak reference. + */ + private WeakReference weakReference() { + return weakRef; + } + /** * Clears weak reference. */ diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/GridCacheSetImpl.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/GridCacheSetImpl.java index 23eca5290266e..cacc9334d8bb4 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/GridCacheSetImpl.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/GridCacheSetImpl.java @@ -66,8 +66,8 @@ public class GridCacheSetImpl extends AbstractCollection implements GridCa /** Removed flag. */ private volatile boolean rmvd; - /** Query storage */ - private final IgniteQueryStorage queryStorage; + /** Iterator holder. */ + private final CacheWeakQueryIteratorsHolder> itHolder; /** * @param ctx Cache context. @@ -85,7 +85,15 @@ public GridCacheSetImpl(GridCacheContext ctx, String name, GridCacheSetHeader hd hdrPart = ctx.affinity().partition(new GridCacheSetHeaderKey(name)); - queryStorage = new IgniteQueryStorage(ctx); + itHolder = new CacheWeakQueryIteratorsHolder>(ctx.logger(GridCacheSetImpl.class)) { + @Override protected T convert(Map.Entry e) { + return e.getKey(); + } + + @Override protected void remove(T item) { + GridCacheSetImpl.this.remove(item); + } + }; } /** {@inheritDoc} */ @@ -340,10 +348,10 @@ private GridCloseableIterator iterator0() { qry.projection(ctx.grid().forNodes(nodes)); - IgniteQueryAbstractStorage.IgniteIterator it = queryStorage.iterator(qry.execute()); + CacheWeakQueryIteratorsHolder.WeakQueryFutureIterator it = itHolder.iterator(qry.execute()); if (rmvd) { - queryStorage.removeIterator(it); + itHolder.removeIterator(it); checkRemoved(); } @@ -430,7 +438,7 @@ void removed(boolean rmvd) { this.rmvd = rmvd; if (rmvd) - queryStorage.clearQueries(); + itHolder.clearQueries(); } /** @@ -445,7 +453,7 @@ private void checkRemoved() { * Checks if set was removed and handles iterators weak reference queue. */ private void onAccess() { - queryStorage.checkWeakQueue(); + itHolder.checkWeakQueue(); checkRemoved(); } @@ -477,28 +485,6 @@ private GridCacheSetItemKey itemKey(Object item) { return S.toString(GridCacheSetImpl.class, this); } - /** - * Queries' storage. - */ - private class IgniteQueryStorage extends IgniteQueryAbstractStorage> { - /** - * @param ctx Cache context. - */ - public IgniteQueryStorage(GridCacheContext ctx) { - super(ctx); - } - - /** {@inheritDoc} */ - @Override protected T convert(Map.Entry v) { - return v != null ? (T) v.getKey() : null; - } - - /** {@inheritDoc} */ - @Override protected void remove(T item) { - GridCacheSetImpl.this.remove(item); - } - } - /** * */ diff --git a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java index 03a60cbaea1db..a44acd1f82a21 100644 --- a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java +++ b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java @@ -5256,11 +5256,13 @@ protected List primaryKeysForCache(GridCacheProjection public void testIgniteCacheIterator() throws Exception { IgniteCache cache = jcache(0); - final int cacheSz = 100; + assertFalse(cache.iterator().hasNext()); - Map entries = new HashMap(); + final int SIZE = 100; - for (int i = 0; i < cacheSz; ++i) { + Map entries = new HashMap<>(); + + for (int i = 0; i < SIZE; ++i) { cache.put(Integer.toString(i), i); entries.put(Integer.toString(i), i); @@ -5301,11 +5303,12 @@ private void checkIteratorRemove(IgniteCache cache, Map> iter = jcache(0).iterator(); + final Iterator> iter = jcache(0).iterator(); assertTrue(iter.hasNext()); @@ -5313,13 +5316,13 @@ private void checkIteratorRemove(IgniteCache cache, Map() { + @Override public Void call() throws Exception { + iter.remove(); - fail(); - } - catch (IllegalStateException e) { - } + return null; + } + }, IllegalStateException.class, null); } /** From ae4cbca3286fcfe1c3fd91d1241d9b143cb29ef6 Mon Sep 17 00:00:00 2001 From: sboikov Date: Fri, 16 Jan 2015 12:06:48 +0300 Subject: [PATCH 13/36] # ignite-53 fail query is there are no data nodes --- .../kernal/processors/cache/query/GridCacheQueryAdapter.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryAdapter.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryAdapter.java index ab27191679c60..65e5cd2350d04 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryAdapter.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryAdapter.java @@ -24,6 +24,7 @@ import org.gridgain.grid.cache.*; import org.gridgain.grid.cache.query.*; import org.gridgain.grid.kernal.processors.cache.*; +import org.gridgain.grid.util.future.*; import org.gridgain.grid.util.typedef.*; import org.gridgain.grid.util.typedef.internal.*; import org.jetbrains.annotations.*; @@ -419,6 +420,9 @@ private GridCacheQueryFuture execute(@Nullable IgniteReducer rmtRed cctx.checkSecurity(GridSecurityPermission.CACHE_READ); + if (nodes.isEmpty()) + return new GridCacheQueryErrorFuture<>(cctx.kernalContext(), new ClusterGroupEmptyException()); + if (log.isDebugEnabled()) log.debug("Executing query [query=" + this + ", nodes=" + nodes + ']'); From aca475f0729690a0bfb8bc2a7cf74b15be668022 Mon Sep 17 00:00:00 2001 From: Yakov Zhdanov Date: Mon, 19 Jan 2015 13:44:13 +0300 Subject: [PATCH 14/36] ignite-53: merge from ignite-1 --- .../cache/GridCacheAbstractFullApiSelfTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java index 9e4a368151e0d..849e4fce96ef0 100644 --- a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java +++ b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java @@ -23,9 +23,9 @@ import org.apache.ignite.configuration.*; import org.apache.ignite.events.*; import org.apache.ignite.lang.*; +import org.apache.ignite.spi.swapspace.inmemory.*; import org.apache.ignite.transactions.*; import org.gridgain.grid.cache.*; -import org.apache.ignite.spi.swapspace.inmemory.*; import org.gridgain.grid.kernal.*; import org.gridgain.grid.util.lang.*; import org.gridgain.grid.util.typedef.*; @@ -42,12 +42,12 @@ import java.util.concurrent.locks.*; import static java.util.concurrent.TimeUnit.*; -import static org.gridgain.grid.cache.GridCacheMode.*; -import static org.gridgain.grid.cache.GridCachePeekMode.*; +import static org.apache.ignite.events.IgniteEventType.*; import static org.apache.ignite.transactions.IgniteTxConcurrency.*; import static org.apache.ignite.transactions.IgniteTxIsolation.*; import static org.apache.ignite.transactions.IgniteTxState.*; -import static org.apache.ignite.events.IgniteEventType.*; +import static org.gridgain.grid.cache.GridCacheMode.*; +import static org.gridgain.grid.cache.GridCachePeekMode.*; import static org.gridgain.testframework.GridTestUtils.*; /** From d80340035818ae26c240694909d9e359c40823ab Mon Sep 17 00:00:00 2001 From: Yakov Zhdanov Date: Mon, 19 Jan 2015 14:27:41 +0300 Subject: [PATCH 15/36] ignite-53: merge from ignite-1 --- .../ignite/internal/processors/cache/IgniteCacheProxy.java | 5 +++-- .../grid/kernal/processors/cache/CacheEntryImpl.java | 2 +- .../processors/cache/CacheWeakQueryIteratorsHolder.java | 7 +++++-- .../processors/cache/query/GridCacheQueryAdapter.java | 1 - 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index 6e8266036464d..3ac11be4f0977 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -74,7 +74,8 @@ public class IgniteCacheProxy extends IgniteAsyncSupportAdapter implements public IgniteCacheProxy(GridCacheContext ctx, GridCacheProjectionEx delegate, @Nullable GridCacheProjectionImpl prj, - boolean async) { + boolean async + ) { super(async); assert ctx != null; @@ -84,7 +85,7 @@ public IgniteCacheProxy(GridCacheContext ctx, this.delegate = delegate; this.prj = prj; - this.itHolder = new CacheWeakQueryIteratorsHolder, Map.Entry>(ctx.logger(IgniteCacheProxy.class)) { + itHolder = new CacheWeakQueryIteratorsHolder, Map.Entry>(ctx.logger(IgniteCacheProxy.class)) { @Override protected Entry convert(Map.Entry e) { return new CacheEntryImpl<>(e.getKey(), e.getValue()); } diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheEntryImpl.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheEntryImpl.java index 3a7a546f81720..c7424707b8da1 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheEntryImpl.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheEntryImpl.java @@ -52,7 +52,7 @@ public CacheEntryImpl(K key, V val) { @SuppressWarnings("unchecked") @Override public T unwrap(Class cls) { if (!cls.equals(getClass())) - throw new IllegalArgumentException("Unwrapping is not supported: " + cls); + throw new IllegalArgumentException("Unwrapping to class is not supported: " + cls); return (T)this; } diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheWeakQueryIteratorsHolder.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheWeakQueryIteratorsHolder.java index eaa65be7a8755..1459927ae7b63 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheWeakQueryIteratorsHolder.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheWeakQueryIteratorsHolder.java @@ -55,7 +55,9 @@ public CacheWeakQueryIteratorsHolder(IgniteLogger log) { public WeakQueryFutureIterator iterator(GridCacheQueryFuture fut) { WeakQueryFutureIterator it = new WeakQueryFutureIterator(fut); - futs.put(it.weakReference(), fut); + GridCacheQueryFuture old = futs.put(it.weakReference(), fut); + + assert old == null; return it; } @@ -75,7 +77,8 @@ public void removeIterator(WeakQueryFutureIterator it) throws IgniteCheckedExcep * Closes unreachable iterators. */ public void checkWeakQueue() { - for (Reference itRef = refQueue.poll(); itRef != null; itRef = refQueue.poll()) { + for (Reference itRef = refQueue.poll(); itRef != null; + itRef = refQueue.poll()) { try { WeakReference weakRef = (WeakReference)itRef; diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryAdapter.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryAdapter.java index 65e5cd2350d04..ba4512c027311 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryAdapter.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryAdapter.java @@ -24,7 +24,6 @@ import org.gridgain.grid.cache.*; import org.gridgain.grid.cache.query.*; import org.gridgain.grid.kernal.processors.cache.*; -import org.gridgain.grid.util.future.*; import org.gridgain.grid.util.typedef.*; import org.gridgain.grid.util.typedef.internal.*; import org.jetbrains.annotations.*; From 9a5dcb3a4a4c964ff4c53bf57a791170b310ff0c Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Mon, 19 Jan 2015 17:31:18 +0400 Subject: [PATCH 16/36] #IGNITE-53: Move CacheWeakQueryIteratorsHolder to context. --- .../processors/cache/IgniteCacheProxy.java | 28 +++++------ .../cache/CacheIteratorConverter.java | 39 +++++++++++++++ .../cache/CacheWeakQueryIteratorsHolder.java | 50 ++++++++----------- .../processors/cache/GridCacheContext.java | 12 +++++ .../datastructures/GridCacheSetImpl.java | 32 ++++++------ 5 files changed, 100 insertions(+), 61 deletions(-) create mode 100644 modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheIteratorConverter.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index 3ac11be4f0977..46fbfb4104b93 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -25,6 +25,7 @@ import org.apache.ignite.resources.*; import org.gridgain.grid.*; import org.gridgain.grid.cache.*; +import org.gridgain.grid.cache.query.*; import org.gridgain.grid.kernal.*; import org.gridgain.grid.kernal.processors.cache.*; import org.gridgain.grid.util.tostring.*; @@ -62,9 +63,6 @@ public class IgniteCacheProxy extends IgniteAsyncSupportAdapter implements /** Projection. */ private GridCacheProjectionImpl prj; - /** Iterator holder. */ - private final CacheWeakQueryIteratorsHolder, Map.Entry> itHolder; - /** * @param ctx Context. * @param delegate Delegate. @@ -85,16 +83,6 @@ public IgniteCacheProxy(GridCacheContext ctx, this.delegate = delegate; this.prj = prj; - itHolder = new CacheWeakQueryIteratorsHolder, Map.Entry>(ctx.logger(IgniteCacheProxy.class)) { - @Override protected Entry convert(Map.Entry e) { - return new CacheEntryImpl<>(e.getKey(), e.getValue()); - } - - @Override protected void remove(Entry item) { - IgniteCacheProxy.this.remove(item.getKey()); - } - }; - gate = ctx.gate(); } @@ -942,9 +930,19 @@ public void removeAll(Collection keys) { GridCacheProjectionImpl prev = gate.enter(prj); try { - itHolder.checkWeakQueue(); + ctx.itHolder().checkWeakQueue(); + + GridCacheQueryFuture> fut = delegate.queries().createScanQuery(null).execute(); - return itHolder.iterator(delegate.queries().createScanQuery(null).execute()); + return ctx.itHolder().iterator(fut, new CacheIteratorConverter, Map.Entry>() { + @Override protected Entry convert(Map.Entry e) { + return new CacheEntryImpl<>(e.getKey(), e.getValue()); + } + + @Override protected void remove(Entry item) { + IgniteCacheProxy.this.remove(item.getKey()); + } + }); } finally { gate.leave(prev); diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheIteratorConverter.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheIteratorConverter.java new file mode 100644 index 0000000000000..466460dd0d866 --- /dev/null +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheIteratorConverter.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gridgain.grid.kernal.processors.cache; + +/** + * @param Type for iterator. + * @param Type for cache query future. + */ +public abstract class CacheIteratorConverter { + /** + * Converts class V to class T. + * + * @param v Item to convert. + * @return Converted item. + */ + protected abstract T convert(V v); + + /** + * Removes item. + * + * @param item Item to remove. + */ + protected abstract void remove(T item); +} diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheWeakQueryIteratorsHolder.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheWeakQueryIteratorsHolder.java index 1459927ae7b63..2980663335df2 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheWeakQueryIteratorsHolder.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheWeakQueryIteratorsHolder.java @@ -26,10 +26,9 @@ import java.util.*; /** - * @param Type for iterator. * @param Type for cache query future. */ -public abstract class CacheWeakQueryIteratorsHolder { +public class CacheWeakQueryIteratorsHolder { /** Iterators weak references queue. */ private final ReferenceQueue refQueue = new ReferenceQueue<>(); @@ -48,12 +47,13 @@ public CacheWeakQueryIteratorsHolder(IgniteLogger log) { } /** - * Iterator over the cache. - * @param fut Query to iterate - * @return iterator + * @param fut Query to iterate. + * @param convert Cache iterator converter. + * @param Type for the iterator. + * @return Iterator over the cache. */ - public WeakQueryFutureIterator iterator(GridCacheQueryFuture fut) { - WeakQueryFutureIterator it = new WeakQueryFutureIterator(fut); + public WeakQueryFutureIterator iterator(GridCacheQueryFuture fut, CacheIteratorConverter convert) { + WeakQueryFutureIterator it = new WeakQueryFutureIterator(fut, convert); GridCacheQueryFuture old = futs.put(it.weakReference(), fut); @@ -94,7 +94,7 @@ public void checkWeakQueue() { } /** - * Cancel all cache queries + * Cancel all cache queries. */ public void clearQueries(){ for (GridCacheQueryFuture fut : futs.values()) { @@ -109,30 +109,20 @@ public void clearQueries(){ futs.clear(); } - /** - * Converts class V to class T. - * - * @param v Item to convert. - * @return Converted item. - */ - protected abstract T convert(V v); - - /** - * Removes item. - * - * @param item Item to remove. - */ - protected abstract void remove(T item); /** * Iterator based of {@link GridCacheQueryFuture}. + * + * @param Type for iterator. */ - public class WeakQueryFutureIterator extends GridCloseableIteratorAdapter { + public class WeakQueryFutureIterator extends GridCloseableIteratorAdapter { /** Query future. */ private final GridCacheQueryFuture fut; /** Weak reference. */ - private final WeakReference weakRef; + private final WeakReference> weakRef; + + CacheIteratorConverter convert; /** Init flag. */ private boolean init; @@ -146,10 +136,12 @@ public class WeakQueryFutureIterator extends GridCloseableIteratorAdapter { /** * @param fut GridCacheQueryFuture to iterate. */ - WeakQueryFutureIterator(GridCacheQueryFuture fut) { + WeakQueryFutureIterator(GridCacheQueryFuture fut, CacheIteratorConverter convert) { this.fut = fut; this.weakRef = new WeakReference<>(this, refQueue); + + this.convert = convert; } /** {@inheritDoc} */ @@ -169,7 +161,7 @@ public class WeakQueryFutureIterator extends GridCloseableIteratorAdapter { if (futNext == null) clearWeakReference(); - next = futNext != null ? convert(futNext) : null; + next = futNext != null ? convert.convert(futNext) : null; return cur; } @@ -198,7 +190,7 @@ public class WeakQueryFutureIterator extends GridCloseableIteratorAdapter { if (cur == null) throw new IllegalStateException(); - CacheWeakQueryIteratorsHolder.this.remove(cur); + convert.remove(cur); cur = null; } @@ -206,7 +198,7 @@ public class WeakQueryFutureIterator extends GridCloseableIteratorAdapter { /** * @return Iterator weak reference. */ - private WeakReference weakReference() { + private WeakReference> weakReference() { return weakRef; } @@ -226,7 +218,7 @@ private void init() throws IgniteCheckedException { if (!init) { V futNext = fut.next(); - next = futNext != null ? convert(futNext) : null; + next = futNext != null ? convert.convert(futNext) : null; init = true; } diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/GridCacheContext.java index ad65344543054..cde739b525bf0 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/GridCacheContext.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/GridCacheContext.java @@ -186,6 +186,9 @@ public class GridCacheContext implements Externalizable { /** Default expiry policy. */ private ExpiryPolicy expiryPlc; + /** Cache weak query iterator holder. */ + private CacheWeakQueryIteratorsHolder> itHolder; + /** * Empty constructor required for {@link Externalizable}. */ @@ -300,6 +303,8 @@ public GridCacheContext( if (expiryPlc instanceof EternalExpiryPolicy) expiryPlc = null; + + itHolder = new CacheWeakQueryIteratorsHolder(log); } /** @@ -836,6 +841,13 @@ public GridCacheContinuousQueryManager continuousQueries() { return contQryMgr; } + /** + * @return Iterators Holder. + */ + public CacheWeakQueryIteratorsHolder> itHolder() { + return itHolder; + } + /** * @return Swap manager. */ diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/GridCacheSetImpl.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/GridCacheSetImpl.java index cacc9334d8bb4..852aa217ceab8 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/GridCacheSetImpl.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/datastructures/GridCacheSetImpl.java @@ -66,9 +66,6 @@ public class GridCacheSetImpl extends AbstractCollection implements GridCa /** Removed flag. */ private volatile boolean rmvd; - /** Iterator holder. */ - private final CacheWeakQueryIteratorsHolder> itHolder; - /** * @param ctx Cache context. * @param name Set name. @@ -84,16 +81,6 @@ public GridCacheSetImpl(GridCacheContext ctx, String name, GridCacheSetHeader hd cache = ctx.cache(); hdrPart = ctx.affinity().partition(new GridCacheSetHeaderKey(name)); - - itHolder = new CacheWeakQueryIteratorsHolder>(ctx.logger(GridCacheSetImpl.class)) { - @Override protected T convert(Map.Entry e) { - return e.getKey(); - } - - @Override protected void remove(T item) { - GridCacheSetImpl.this.remove(item); - } - }; } /** {@inheritDoc} */ @@ -348,10 +335,21 @@ private GridCloseableIterator iterator0() { qry.projection(ctx.grid().forNodes(nodes)); - CacheWeakQueryIteratorsHolder.WeakQueryFutureIterator it = itHolder.iterator(qry.execute()); + GridCacheQueryFuture> fut = qry.execute(); + + CacheWeakQueryIteratorsHolder.WeakQueryFutureIterator it = + ctx.itHolder().iterator(fut, new CacheIteratorConverter>() { + @Override protected T convert(Map.Entry e) { + return e.getKey(); + } + + @Override protected void remove(T item) { + GridCacheSetImpl.this.remove(item); + } + }); if (rmvd) { - itHolder.removeIterator(it); + ctx.itHolder().removeIterator(it); checkRemoved(); } @@ -438,7 +436,7 @@ void removed(boolean rmvd) { this.rmvd = rmvd; if (rmvd) - itHolder.clearQueries(); + ctx.itHolder().clearQueries(); } /** @@ -453,7 +451,7 @@ private void checkRemoved() { * Checks if set was removed and handles iterators weak reference queue. */ private void onAccess() { - itHolder.checkWeakQueue(); + ctx.itHolder().checkWeakQueue(); checkRemoved(); } From 0b48f37c2f4fed67dc6fa9a827a1a3b37f159ac6 Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Mon, 19 Jan 2015 17:35:34 +0400 Subject: [PATCH 17/36] #IGNITE-53: check weak queue in iterator holder on access. --- .../gridgain/grid/kernal/processors/cache/GridCacheGateway.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/GridCacheGateway.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/GridCacheGateway.java index bb04ae0708418..b6a0ec8e185db 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/GridCacheGateway.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/GridCacheGateway.java @@ -87,6 +87,8 @@ public void leave() { */ @Nullable public GridCacheProjectionImpl enter(@Nullable GridCacheProjectionImpl prj) { try { + ctx.itHolder().checkWeakQueue(); + GridCacheAdapter cache = ctx.cache(); GridCachePreloader preldr = cache != null ? cache.preloader() : null; From eb7453139fa5d1249098920a96863c7640600221 Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Mon, 19 Jan 2015 19:03:23 +0300 Subject: [PATCH 18/36] #IGNITE-53: Add test on clearing iterators. --- .../GridCacheAbstractFullApiSelfTest.java | 55 ++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java index 849e4fce96ef0..d0e8d3420b8ed 100644 --- a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java +++ b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java @@ -18,6 +18,7 @@ package org.gridgain.grid.kernal.processors.cache; import com.google.common.collect.*; +import junit.framework.*; import org.apache.ignite.*; import org.apache.ignite.cluster.*; import org.apache.ignite.configuration.*; @@ -25,8 +26,10 @@ import org.apache.ignite.lang.*; import org.apache.ignite.spi.swapspace.inmemory.*; import org.apache.ignite.transactions.*; +import org.gridgain.grid.*; import org.gridgain.grid.cache.*; import org.gridgain.grid.kernal.*; +import org.gridgain.grid.kernal.processors.cache.query.*; import org.gridgain.grid.util.lang.*; import org.gridgain.grid.util.typedef.*; import org.gridgain.grid.util.typedef.internal.*; @@ -5084,7 +5087,7 @@ public void testIgniteCacheIterator() throws Exception { assertFalse(cache.iterator().hasNext()); - final int SIZE = 100; + final int SIZE = 1000; Map entries = new HashMap<>(); @@ -5100,6 +5103,8 @@ public void testIgniteCacheIterator() throws Exception { checkIteratorRemove(cache, entries); + checkIteratorEmpty(cache); + } /** @@ -5202,6 +5207,54 @@ private void checkIteratorCache(IgniteCache cache, Map cache) throws GridInterruptedException, InterruptedException { + int cnt = 5; + + for (int i = 0; i < cnt; ++i) { + Iterator> iter = cache.iterator(); + + iter.next(); + + assert iter.hasNext(); + } + + System.gc(); + + for (int i = 0; i < 10; i++) { + try { + cache.size(); // Trigger weak queue poll. + + checkIteratorsCleared(); + } + catch (AssertionFailedError e) { + if (i == 9) + throw e; + + log.info("Set iterators not cleared, will wait"); + + Thread.sleep(500); + } + } + } + /** * @param cache Cache. * @param cnt Keys count. From 222721bfe5637af4e97a85c1f6c69b4561350a80 Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Mon, 19 Jan 2015 19:45:51 +0300 Subject: [PATCH 19/36] #IGNITE-53: Change log on U.log --- .../processors/cache/CacheWeakQueryIteratorsHolder.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheWeakQueryIteratorsHolder.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheWeakQueryIteratorsHolder.java index 2980663335df2..f955695e2c2d3 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheWeakQueryIteratorsHolder.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheWeakQueryIteratorsHolder.java @@ -20,6 +20,7 @@ import org.apache.ignite.*; import org.gridgain.grid.cache.query.*; import org.gridgain.grid.util.*; +import org.gridgain.grid.util.typedef.internal.*; import org.jdk8.backport.*; import java.lang.ref.*; @@ -88,7 +89,7 @@ public void checkWeakQueue() { fut.cancel(); } catch (IgniteCheckedException e) { - log.error("Failed to close iterator.", e); + U.error(log, "Failed to close iterator.", e); } } } @@ -102,7 +103,7 @@ public void clearQueries(){ fut.cancel(); } catch (IgniteCheckedException e) { - log.error("Failed to close iterator.", e); + U.error(log, "Failed to close iterator.", e); } } From c500eda65bb2aa96360027b96dac0169c8c979dc Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Tue, 20 Jan 2015 11:27:36 +0300 Subject: [PATCH 20/36] #IGNITE-53: wip --- .../processors/cache/IgniteCacheProxy.java | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index 9d9392ad8a6b2..fbf69042bf7c0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -946,15 +946,33 @@ public void removeAll(Collection keys) { } /** {@inheritDoc} */ - @Override public void registerCacheEntryListener(CacheEntryListenerConfiguration cacheEntryLsnrConfiguration) { - // TODO IGNITE-1. - throw new UnsupportedOperationException(); + @Override public void registerCacheEntryListener(CacheEntryListenerConfiguration lsnrCfg) { + GridCacheProjectionImpl prev = gate.enter(prj); + + try { + ctx.continuousQueries().registerCacheEntryListener(lsnrCfg, true); + } + catch (IgniteCheckedException e) { + throw cacheException(e); + } + finally { + gate.leave(prev); + } } /** {@inheritDoc} */ - @Override public void deregisterCacheEntryListener(CacheEntryListenerConfiguration cacheEntryLsnrConfiguration) { - // TODO IGNITE-1. - throw new UnsupportedOperationException(); + @Override public void deregisterCacheEntryListener(CacheEntryListenerConfiguration lsnrCfg) { + GridCacheProjectionImpl prev = gate.enter(prj); + + try { + ctx.continuousQueries().deregisterCacheEntryListener(lsnrCfg); + } + catch (IgniteCheckedException e) { + throw cacheException(e); + } + finally { + gate.leave(prev); + } } /** {@inheritDoc} */ From 83a571bc8d5bea30d2b81114857ebaad25972961 Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Tue, 20 Jan 2015 14:48:59 +0300 Subject: [PATCH 21/36] #IGNITE-53: wip --- .../internal/processors/cache/IgniteCacheProxy.java | 8 +++++++- .../cache/GridCacheAbstractFullApiSelfTest.java | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index fbf69042bf7c0..e3f60dfea5c4d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -982,7 +982,13 @@ public void removeAll(Collection keys) { try { ctx.itHolder().checkWeakQueue(); - GridCacheQueryFuture> fut = delegate.queries().createScanQuery(null).execute(); + GridCacheQuery> query = delegate.queries().createScanQuery(null); + + query.includeBackups(false); + query.enableDedup(true); + query.keepAll(false); + + GridCacheQueryFuture> fut = query.execute(); return ctx.itHolder().iterator(fut, new CacheIteratorConverter, Map.Entry>() { @Override protected Entry convert(Map.Entry e) { diff --git a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java index d0e8d3420b8ed..7367cb88df56e 100644 --- a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java +++ b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java @@ -5087,7 +5087,7 @@ public void testIgniteCacheIterator() throws Exception { assertFalse(cache.iterator().hasNext()); - final int SIZE = 1000; + final int SIZE = 20000; Map entries = new HashMap<>(); From a5453695c1993d245c0f67ecdfe95f79486f8f24 Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Wed, 21 Jan 2015 17:20:17 +0300 Subject: [PATCH 22/36] #IGNITE-66: change gridgain to ignite in examples/pom-standalone.xml --- examples/pom-standalone.xml | 50 ++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/examples/pom-standalone.xml b/examples/pom-standalone.xml index 0258190f51ac5..eeb02cf09ebb9 100644 --- a/examples/pom-standalone.xml +++ b/examples/pom-standalone.xml @@ -28,55 +28,55 @@ UTF-8 - 6.5.6 + 6.5.6 src/main/java - org.gridgain - gridgain-examples - ${gridgain.version} + org.apache.ignite + ignite-examples + ${ignite.version} - org.gridgain - gridgain-core - ${gridgain.version} + org.apache.ignite + ignite-core + ${ignite.version} - org.gridgain - gridgain-hibernate - ${gridgain.version} + org.apache.ignite + ignite-hibernate + ${ignite.version} - org.gridgain - gridgain-scalar - ${gridgain.version} + org.apache.ignite + ignite-scalar + ${ignite.version} - org.gridgain - gridgain-spring - ${gridgain.version} + org.apache.ignite + ignite-spring + ${ignite.version} - org.gridgain - gridgain-log4j - ${gridgain.version} + org.apache.ignite + ignite-log4j + ${ignite.version} - org.gridgain - gridgain-indexing - ${gridgain.version} + org.apache.ignite + ignite-indexing + ${ignite.version} - org.gridgain - gridgain-schedule - ${gridgain.version} + org.apache.ignite + ignite-schedule + ${ignite.version} From e8de0fce2a4cb35eb9f82ee22de18053bb3e6ae5 Mon Sep 17 00:00:00 2001 From: Yakov Zhdanov Date: Wed, 21 Jan 2015 20:30:15 +0300 Subject: [PATCH 23/36] ignite-53 review --- .../processors/cache/IgniteCacheProxy.java | 3 --- .../cache/query/GridCacheQueryFutureAdapter.java | 15 ++++++++------- .../cache/query/GridCacheQueryManager.java | 2 +- .../cache/GridCacheAbstractFullApiSelfTest.java | 1 - 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index 7a1dd8f6259b1..fadb7befa3765 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -878,12 +878,9 @@ public void removeAll(Collection keys) { GridCacheProjectionImpl prev = gate.enter(prj); try { - ctx.itHolder().checkWeakQueue(); - GridCacheQuery> query = delegate.queries().createScanQuery(null); query.includeBackups(false); - query.enableDedup(true); query.keepAll(false); GridCacheQueryFuture> fut = query.execute(); diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryFutureAdapter.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryFutureAdapter.java index 82a9a667e62df..9c5c0ad59d70f 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryFutureAdapter.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryFutureAdapter.java @@ -23,6 +23,7 @@ import org.gridgain.grid.cache.query.*; import org.gridgain.grid.kernal.processors.cache.*; import org.gridgain.grid.kernal.processors.timeout.*; +import org.gridgain.grid.util.*; import org.gridgain.grid.util.future.*; import org.gridgain.grid.util.typedef.*; import org.gridgain.grid.util.typedef.internal.*; @@ -59,7 +60,7 @@ public abstract class GridCacheQueryFutureAdapter extends GridFutureAda protected final GridCacheQueryBean qry; /** Set of received keys used to deduplicate query result set. */ - private final Collection keys = new HashSet<>(); + private Collection keys; /** */ private final Queue> queue = new LinkedList<>(); @@ -122,6 +123,8 @@ protected GridCacheQueryFutureAdapter(GridCacheContext cctx, GridCacheQuer cctx.time().addTimeoutObject(this); } + + keys = qry.query().enableDedup() ? new GridConcurrentHashSet() : null; } /** @@ -336,13 +339,11 @@ private Collection dedupIfRequired(Collection col) { if (!qry.query().enableDedup()) return col; - Collection dedupCol = new LinkedList<>(); + Collection dedupCol = new ArrayList<>(col.size()); - synchronized (mux) { - for (Object o : col) - if (!(o instanceof Map.Entry) || keys.add(((Map.Entry)o).getKey())) - dedupCol.add(o); - } + for (Object o : col) + if (!(o instanceof Map.Entry) || keys.add(((Map.Entry)o).getKey())) + dedupCol.add(o); return dedupCol; } diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryManager.java index acdc6ecb14474..b65c566509573 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryManager.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryManager.java @@ -757,7 +757,7 @@ private GridCloseableIterator> scanIterator(final GridCacheQ GridIterator> heapIt = new GridIteratorAdapter>() { private IgniteBiTuple next; - private Iterator iter = prj.keySet().iterator(); + private Iterator iter = qry.includeBackups() ? prj.keySet().iterator() : prj.primaryKeySet().iterator(); { advance(); diff --git a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java index 6b4f5aa5b7d72..b29fe671e1d56 100644 --- a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java +++ b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java @@ -5105,7 +5105,6 @@ public void testIgniteCacheIterator() throws Exception { checkIteratorRemove(cache, entries); checkIteratorEmpty(cache); - } /** From 78ce7af867c0cfd37e10508e571d58164689027c Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Thu, 22 Jan 2015 13:24:16 +0300 Subject: [PATCH 24/36] #IGNITE-53: fix for replicated cache. --- .../internal/processors/cache/IgniteCacheProxy.java | 9 +++++---- .../GridCacheReplicatedPreloadLifecycleSelfTest.java | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index fadb7befa3765..e23beb60c3862 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -878,12 +878,13 @@ public void removeAll(Collection keys) { GridCacheProjectionImpl prev = gate.enter(prj); try { - GridCacheQuery> query = delegate.queries().createScanQuery(null); + GridCacheQuery> qry = delegate.queries().createScanQuery(null); - query.includeBackups(false); - query.keepAll(false); + qry.includeBackups(ctx.config().getCacheMode() == GridCacheMode.REPLICATED); - GridCacheQueryFuture> fut = query.execute(); + qry.keepAll(false); + + GridCacheQueryFuture> fut = qry.execute(); return ctx.itHolder().iterator(fut, new CacheIteratorConverter, Map.Entry>() { @Override protected Entry convert(Map.Entry e) { diff --git a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/distributed/replicated/preloader/GridCacheReplicatedPreloadLifecycleSelfTest.java b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/distributed/replicated/preloader/GridCacheReplicatedPreloadLifecycleSelfTest.java index 3e65a520e7ec0..ea28da46764b0 100644 --- a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/distributed/replicated/preloader/GridCacheReplicatedPreloadLifecycleSelfTest.java +++ b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/distributed/replicated/preloader/GridCacheReplicatedPreloadLifecycleSelfTest.java @@ -162,7 +162,6 @@ public void checkCache(Object[] keys) throws Exception { } } - /** * @param keys Keys. * @throws Exception If failed. @@ -182,6 +181,8 @@ public void checkScanQuery(Object[] keys) throws Exception { GridCacheQuery> qry = c2.queries().createScanQuery(null); + qry.includeBackups(true); + final int i0 = j; final int j0 = i; From 84aba1f5b850cbf7205e1d57b96e804e55bf0e4a Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Thu, 22 Jan 2015 18:56:33 +0300 Subject: [PATCH 25/36] #IGNITE-53: wip. --- .../processors/cache/query/GridCacheQueryAdapter.java | 2 +- .../cache/query/GridCacheQueryFutureAdapter.java | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryAdapter.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryAdapter.java index ba4512c027311..ebfa554ad27d1 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryAdapter.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryAdapter.java @@ -469,7 +469,7 @@ private Collection nodes() { return Collections.singletonList(cctx.localNode()); case REPLICATED: - if (prj != null) + if (prj != null || !incBackups) return nodes(cctx, prj); GridCacheDistributionMode mode = cctx.config().getDistributionMode(); diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryFutureAdapter.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryFutureAdapter.java index 9c5c0ad59d70f..97317adc3f2bc 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryFutureAdapter.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryFutureAdapter.java @@ -23,7 +23,6 @@ import org.gridgain.grid.cache.query.*; import org.gridgain.grid.kernal.processors.cache.*; import org.gridgain.grid.kernal.processors.timeout.*; -import org.gridgain.grid.util.*; import org.gridgain.grid.util.future.*; import org.gridgain.grid.util.typedef.*; import org.gridgain.grid.util.typedef.internal.*; @@ -124,7 +123,7 @@ protected GridCacheQueryFutureAdapter(GridCacheContext cctx, GridCacheQuer cctx.time().addTimeoutObject(this); } - keys = qry.query().enableDedup() ? new GridConcurrentHashSet() : null; + keys = qry.query().enableDedup() ? new HashSet() : null; } /** @@ -341,9 +340,11 @@ private Collection dedupIfRequired(Collection col) { Collection dedupCol = new ArrayList<>(col.size()); - for (Object o : col) - if (!(o instanceof Map.Entry) || keys.add(((Map.Entry)o).getKey())) - dedupCol.add(o); + synchronized (mux) { + for (Object o : col) + if (!(o instanceof Map.Entry) || keys.add(((Map.Entry) o).getKey())) + dedupCol.add(o); + } return dedupCol; } From a8f87668472e732cfc0a664acbb180087da954e0 Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Thu, 22 Jan 2015 20:42:36 +0300 Subject: [PATCH 26/36] #IGNITE-53: wip. --- .../kernal/processors/cache/query/GridCacheQueryAdapter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryAdapter.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryAdapter.java index ebfa554ad27d1..ba4512c027311 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryAdapter.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryAdapter.java @@ -469,7 +469,7 @@ private Collection nodes() { return Collections.singletonList(cctx.localNode()); case REPLICATED: - if (prj != null || !incBackups) + if (prj != null) return nodes(cctx, prj); GridCacheDistributionMode mode = cctx.config().getDistributionMode(); From fbf6e60aeb6bad52c1473485cc4b5d267240b530 Mon Sep 17 00:00:00 2001 From: Yakov Zhdanov Date: Thu, 22 Jan 2015 21:14:07 +0300 Subject: [PATCH 27/36] Ignite-53 review --- .../internal/processors/cache/IgniteCacheProxy.java | 9 +++------ .../processors/cache/query/GridCacheQueryAdapter.java | 11 ++--------- .../processors/cache/query/GridCacheQueryManager.java | 3 ++- .../cache/GridCacheAbstractFullApiSelfTest.java | 10 +++++++--- 4 files changed, 14 insertions(+), 19 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index fadb7befa3765..51d03f48bb661 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -878,12 +878,9 @@ public void removeAll(Collection keys) { GridCacheProjectionImpl prev = gate.enter(prj); try { - GridCacheQuery> query = delegate.queries().createScanQuery(null); - - query.includeBackups(false); - query.keepAll(false); - - GridCacheQueryFuture> fut = query.execute(); + GridCacheQueryFuture> fut = delegate.queries().createScanQuery(null) + .keepAll(false) + .execute(); return ctx.itHolder().iterator(fut, new CacheIteratorConverter, Map.Entry>() { @Override protected Entry convert(Map.Entry e) { diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryAdapter.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryAdapter.java index ba4512c027311..38ecc74fd5615 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryAdapter.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryAdapter.java @@ -65,13 +65,13 @@ public class GridCacheQueryAdapter implements GridCacheQuery { private volatile GridCacheQueryMetricsAdapter metrics; /** */ - private volatile int pageSize; + private volatile int pageSize = DFLT_PAGE_SIZE; /** */ private volatile long timeout; /** */ - private volatile boolean keepAll; + private volatile boolean keepAll = true; /** */ private volatile boolean incBackups; @@ -123,13 +123,6 @@ public GridCacheQueryAdapter(GridCacheContext cctx, log = cctx.logger(getClass()); - pageSize = DFLT_PAGE_SIZE; - timeout = 0; - keepAll = true; - incBackups = false; - dedup = false; - prj = null; - metrics = new GridCacheQueryMetricsAdapter(); } diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryManager.java index b65c566509573..adedf89110182 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryManager.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/query/GridCacheQueryManager.java @@ -757,7 +757,8 @@ private GridCloseableIterator> scanIterator(final GridCacheQ GridIterator> heapIt = new GridIteratorAdapter>() { private IgniteBiTuple next; - private Iterator iter = qry.includeBackups() ? prj.keySet().iterator() : prj.primaryKeySet().iterator(); + private Iterator iter = qry.includeBackups() || cctx.isReplicated() ? + prj.keySet().iterator() : prj.primaryKeySet().iterator(); { advance(); diff --git a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java index b29fe671e1d56..cf4dca044563a 100644 --- a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java +++ b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java @@ -5092,10 +5092,14 @@ public void testIgniteCacheIterator() throws Exception { Map entries = new HashMap<>(); - for (int i = 0; i < SIZE; ++i) { - cache.put(Integer.toString(i), i); + try (IgniteDataLoader dataLoader = ignite(0).dataLoader(null)) { + dataLoader.isolated(true); - entries.put(Integer.toString(i), i); + for (int i = 0; i < SIZE; ++i) { + dataLoader.addData(Integer.toString(i), i); + + entries.put(Integer.toString(i), i); + } } checkIteratorHasNext(); From bdff6c598df00265e17747f2d063bba0e4a5e789 Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Fri, 23 Jan 2015 10:24:19 +0300 Subject: [PATCH 28/36] #IGNITE-53: Move implementation from IgniteCacheProxy to GridCacheAdapter. --- .../processors/cache/IgniteCacheProxy.java | 14 +--------- .../processors/cache/GridCacheAdapter.java | 26 ++++++++++++++++++- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index d143f751a9b9c..2bad25d3676c6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -827,19 +827,7 @@ else if (clazz.equals(Ignite.class)) GridCacheProjectionImpl prev = gate.enter(prj); try { - GridCacheQueryFuture> fut = delegate.queries().createScanQuery(null) - .keepAll(false) - .execute(); - - return ctx.itHolder().iterator(fut, new CacheIteratorConverter, Map.Entry>() { - @Override protected Entry convert(Map.Entry e) { - return new CacheEntryImpl<>(e.getKey(), e.getValue()); - } - - @Override protected void remove(Entry item) { - IgniteCacheProxy.this.remove(item.getKey()); - } - }); + return ((GridCacheAdapter)delegate).igniteIterator(); } finally { gate.leave(prev); diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/GridCacheAdapter.java index 4fdfc193a56d3..e1459bf9ca15d 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/GridCacheAdapter.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/GridCacheAdapter.java @@ -22,7 +22,6 @@ import org.apache.ignite.cluster.*; import org.apache.ignite.compute.*; import org.apache.ignite.configuration.*; -import org.apache.ignite.dataload.*; import org.apache.ignite.fs.*; import org.apache.ignite.lang.*; import org.apache.ignite.plugin.security.*; @@ -52,6 +51,7 @@ import org.jdk8.backport.*; import org.jetbrains.annotations.*; +import javax.cache.*; import javax.cache.expiry.*; import javax.cache.processor.*; import java.io.*; @@ -3618,6 +3618,30 @@ public void localLoad(Collection keys) throws IgniteCheckedExceptio return entrySet().iterator(); } + /** + * @return Distributed ignite cache iterator. + */ + public Iterator> igniteIterator() { + GridCacheQueryFuture> fut = queries().createScanQuery(null) + .keepAll(false) + .execute(); + + return ctx.itHolder().iterator(fut, new CacheIteratorConverter, Map.Entry>() { + @Override protected Cache.Entry convert(Map.Entry e) { + return new CacheEntryImpl<>(e.getKey(), e.getValue()); + } + + @Override protected void remove(Cache.Entry item) { + try { + GridCacheAdapter.this.removex(item.getKey()); + } + catch (IgniteCheckedException e) { + throw new CacheException(e); + } + } + }); + } + /** {@inheritDoc} */ @Nullable @Override public V promote(K key) throws IgniteCheckedException { return promote(key, true); From c0006f6c8d8a659a2545317b4eb98d0fc5fb9683 Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Fri, 23 Jan 2015 11:04:45 +0300 Subject: [PATCH 29/36] #IGNITE-53: Move implementation from IgniteCacheProxy to GridCacheAdapter. --- .../processors/cache/IgniteCacheProxy.java | 2 +- .../processors/cache/GridCacheAdapter.java | 32 +++++++++++-------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index 2bad25d3676c6..a13f74843cba1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -827,7 +827,7 @@ else if (clazz.equals(Ignite.class)) GridCacheProjectionImpl prev = gate.enter(prj); try { - return ((GridCacheAdapter)delegate).igniteIterator(); + return ((GridCacheAdapter)delegate).igniteIterator(prj); } finally { gate.leave(prev); diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/GridCacheAdapter.java index e1459bf9ca15d..bd735f46df14b 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/GridCacheAdapter.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/GridCacheAdapter.java @@ -1379,7 +1379,7 @@ private boolean evictx(K key, GridCacheVersion ver, String taskName = ctx.kernalContext().job().currentTaskName(); return getAllAsync(Collections.singletonList(key), /*force primary*/true, /*skip tx*/false, null, null, - taskName, true).chain(new CX1>, V>() { + taskName, true).chain(new CX1>, V>() { @Override public V applyx(IgniteFuture> e) throws IgniteCheckedException { return e.get().get(key); } @@ -2234,12 +2234,12 @@ public String toString() { return syncOp(new SyncOp>(true) { @Nullable @Override public EntryProcessorResult op(IgniteTxLocalAdapter tx) - throws IgniteCheckedException { + throws IgniteCheckedException { Map> invokeMap = Collections.singletonMap(key, (EntryProcessor)entryProcessor); IgniteFuture>>> fut = - tx.invokeAsync(ctx, invokeMap, args); + tx.invokeAsync(ctx, invokeMap, args); Map> resMap = fut.get().value(); @@ -2313,7 +2313,7 @@ public String toString() { return fut0.chain(new CX1>>>, EntryProcessorResult>() { @Override public EntryProcessorResult applyx(IgniteFuture>>> fut) - throws IgniteCheckedException { + throws IgniteCheckedException { GridCacheReturn>> ret = fut.get(); Map> resMap = ret.value(); @@ -2421,7 +2421,7 @@ public String toString() { return syncOp(new SyncOp>>(map.size() == 1) { @Nullable @Override public Map> op(IgniteTxLocalAdapter tx) - throws IgniteCheckedException { + throws IgniteCheckedException { IgniteFuture>>> fut = tx.invokeAsync(ctx, map, args); return fut.get().value(); @@ -2661,7 +2661,7 @@ public String toString() { ctx.deploy().registerClass(oldVal); return tx.putAllAsync(ctx, F.t(key, newVal), false, null, -1, ctx.equalsPeekArray(oldVal)).get() - .success(); + .success(); } @Override public String toString() { @@ -2804,7 +2804,7 @@ public String toString() { @Override public IgniteFuture op(IgniteTxLocalAdapter tx) { // TODO should we invoke interceptor here? return tx.removeAllAsync(ctx, Collections.singletonList(key), null, true, filter) - .chain((IgniteClosure>, V>) RET2VAL); + .chain((IgniteClosure>, V>) RET2VAL); } @Override public String toString() { @@ -2940,7 +2940,7 @@ public String toString() { ctx.deploy().registerClass(val); return tx.removeAllAsync(ctx, Collections.singletonList(key), null, true, - ctx.vararg(F.cacheContainsPeek(val))).get(); + ctx.vararg(F.cacheContainsPeek(val))).get(); } @Override public String toString() { @@ -3035,7 +3035,7 @@ public String toString() { } return tx.removeAllAsync(ctx, Collections.singletonList(key), null, true, - ctx.vararg(F.cacheContainsPeek(val))); + ctx.vararg(F.cacheContainsPeek(val))); } @Override public String toString() { @@ -3096,7 +3096,7 @@ public String toString() { key0 = (K)ctx.marshalToPortable(key); return tx.removeAllAsync(ctx, Collections.singletonList(key0), null, false, - ctx.vararg(F.cacheContainsPeek(val))).get().success(); + ctx.vararg(F.cacheContainsPeek(val))).get().success(); } @Override public String toString() { @@ -3140,7 +3140,7 @@ public String toString() { } return tx.removeAllAsync(ctx, Collections.singletonList(key0), null, false, - ctx.vararg(F.cacheContainsPeek(val))).chain( + ctx.vararg(F.cacheContainsPeek(val))).chain( (IgniteClosure>, Boolean>)RET2FLAG); } @@ -3340,7 +3340,7 @@ public GridCacheMetricsAdapter metrics0() { @Override public IgniteTx txStartPartition(int partId, IgniteTxConcurrency concurrency, IgniteTxIsolation isolation, long timeout, int txSize) throws IllegalStateException, IgniteCheckedException { return ctx.kernalContext().cache().transactions().txStartPartition(name(), partId, concurrency, isolation, - timeout, txSize); + timeout, txSize); } /** {@inheritDoc} */ @@ -3619,9 +3619,10 @@ public void localLoad(Collection keys) throws IgniteCheckedExceptio } /** + * @param prj Projection. * @return Distributed ignite cache iterator. */ - public Iterator> igniteIterator() { + public Iterator> igniteIterator(final GridCacheProjectionImpl prj) { GridCacheQueryFuture> fut = queries().createScanQuery(null) .keepAll(false) .execute(); @@ -3632,12 +3633,17 @@ public Iterator> igniteIterator() { } @Override protected void remove(Cache.Entry item) { + GridCacheProjectionImpl prev = ctx.gate().enter(prj); + try { GridCacheAdapter.this.removex(item.getKey()); } catch (IgniteCheckedException e) { throw new CacheException(e); } + finally { + ctx.gate().leave(prev); + } } }); } From fe11163627df0246275a022a9bf3337e19f42649 Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Fri, 23 Jan 2015 14:01:02 +0300 Subject: [PATCH 30/36] #IGNITE-53: wip --- .../ignite/internal/processors/cache/IgniteCacheProxy.java | 2 -- .../processors/cache/GridCacheAbstractFullApiSelfTest.java | 6 +++--- .../GridCacheReplicatedPreloadLifecycleSelfTest.java | 2 -- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index a13f74843cba1..5d93a09667509 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -24,8 +24,6 @@ import org.apache.ignite.lang.*; import org.apache.ignite.resources.*; import org.gridgain.grid.cache.*; -import org.gridgain.grid.cache.query.*; -import org.gridgain.grid.kernal.*; import org.gridgain.grid.kernal.processors.cache.*; import org.gridgain.grid.util.tostring.*; import org.gridgain.grid.util.typedef.*; diff --git a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java index 397b259662bf4..1c72fac725838 100644 --- a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java +++ b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java @@ -46,12 +46,12 @@ import java.util.concurrent.locks.*; import static java.util.concurrent.TimeUnit.*; -import static org.apache.ignite.events.IgniteEventType.*; +import static org.gridgain.grid.cache.GridCacheMode.*; +import static org.gridgain.grid.cache.GridCachePeekMode.*; import static org.apache.ignite.transactions.IgniteTxConcurrency.*; import static org.apache.ignite.transactions.IgniteTxIsolation.*; import static org.apache.ignite.transactions.IgniteTxState.*; -import static org.gridgain.grid.cache.GridCacheMode.*; -import static org.gridgain.grid.cache.GridCachePeekMode.*; +import static org.apache.ignite.events.IgniteEventType.*; import static org.gridgain.testframework.GridTestUtils.*; /** diff --git a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/distributed/replicated/preloader/GridCacheReplicatedPreloadLifecycleSelfTest.java b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/distributed/replicated/preloader/GridCacheReplicatedPreloadLifecycleSelfTest.java index 2a1fe88228df8..45094cf28cf8a 100644 --- a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/distributed/replicated/preloader/GridCacheReplicatedPreloadLifecycleSelfTest.java +++ b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/distributed/replicated/preloader/GridCacheReplicatedPreloadLifecycleSelfTest.java @@ -182,8 +182,6 @@ public void checkScanQuery(Object[] keys) throws Exception { GridCacheQuery> qry = c2.queries().createScanQuery(null); - qry.includeBackups(true); - final int i0 = j; final int j0 = i; From 68d01efcfdac7c951074803a2b411ae295e410f0 Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Fri, 23 Jan 2015 16:06:19 +0300 Subject: [PATCH 31/36] #IGNITE-53: fix test --- .../cache/GridCacheAbstractFullApiSelfTest.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java index 397b259662bf4..6450a554e99da 100644 --- a/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java +++ b/modules/core/src/test/java/org/gridgain/grid/kernal/processors/cache/GridCacheAbstractFullApiSelfTest.java @@ -5092,14 +5092,10 @@ public void testIgniteCacheIterator() throws Exception { Map entries = new HashMap<>(); - try (IgniteDataLoader dataLoader = ignite(0).dataLoader(null)) { - dataLoader.isolated(true); + for (int i = 0; i < SIZE; ++i) { + cache.put(Integer.toString(i), i); - for (int i = 0; i < SIZE; ++i) { - dataLoader.addData(Integer.toString(i), i); - - entries.put(Integer.toString(i), i); - } + entries.put(Integer.toString(i), i); } checkIteratorHasNext(); From 3362a61d64b7e54a92788fe300cde010852c1e10 Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Fri, 23 Jan 2015 19:05:39 +0300 Subject: [PATCH 32/36] #IGNITE-53: merge --- .../processors/cache/CacheEntryImpl.java | 8 +- .../processors/cache/GridCacheAdapter.java | 32 ++++ .../processors/cache/GridCacheContext.java | 13 ++ .../processors/cache/GridCacheGateway.java | 2 + .../processors/cache/IgniteCacheProxy.java | 19 +- .../datastructures/GridCacheSetImpl.java | 174 ++--------------- .../cache/query/GridCacheQueryAdapter.java | 14 +- .../query/GridCacheQueryFutureAdapter.java | 7 +- .../cache/query/GridCacheQueryManager.java | 4 +- .../cache/CacheWeakQueryIteratorsHolder.java | 22 +-- .../GridCacheAbstractFullApiSelfTest.java | 178 ++++++++++++++++++ 11 files changed, 270 insertions(+), 203 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryImpl.java index 27dadbd7512c2..e609cd10f4d8b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryImpl.java @@ -49,8 +49,12 @@ public CacheEntryImpl(K key, V val) { } /** {@inheritDoc} */ - @Override public T unwrap(Class clazz) { - throw new IllegalArgumentException(); + @SuppressWarnings("unchecked") + @Override public T unwrap(Class cls) { + if (!cls.equals(getClass())) + throw new IllegalArgumentException("Unwrapping to class is not supported: " + cls); + + return (T)this; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java index 112483a7ba175..3c0ed75732237 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java @@ -46,9 +46,11 @@ import org.apache.ignite.internal.util.tostring.*; import org.apache.ignite.internal.util.typedef.*; import org.apache.ignite.internal.util.typedef.internal.*; +import org.gridgain.grid.kernal.processors.cache.*; import org.jdk8.backport.*; import org.jetbrains.annotations.*; +import javax.cache.*; import javax.cache.expiry.*; import javax.cache.processor.*; import java.io.*; @@ -3615,6 +3617,36 @@ public void localLoad(Collection keys) throws IgniteCheckedExceptio return entrySet().iterator(); } + /** + * @param prj Projection. + * @return Distributed ignite cache iterator. + */ + public Iterator> igniteIterator(final GridCacheProjectionImpl prj) { + CacheQueryFuture> fut = queries().createScanQuery(null) + .keepAll(false) + .execute(); + + return ctx.itHolder().iterator(fut, new CacheIteratorConverter, Map.Entry>() { + @Override protected Cache.Entry convert(Map.Entry e) { + return new CacheEntryImpl<>(e.getKey(), e.getValue()); + } + + @Override protected void remove(Cache.Entry item) { + GridCacheProjectionImpl prev = ctx.gate().enter(prj); + + try { + GridCacheAdapter.this.removex(item.getKey()); + } + catch (IgniteCheckedException e) { + throw new CacheException(e); + } + finally { + ctx.gate().leave(prev); + } + } + }); + } + /** {@inheritDoc} */ @Nullable @Override public V promote(K key) throws IgniteCheckedException { return promote(key, true); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java index 567d0bb2f2d85..d1dcaa3aa9b09 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java @@ -52,6 +52,7 @@ import org.apache.ignite.internal.util.lang.*; import org.apache.ignite.internal.util.offheap.unsafe.*; import org.apache.ignite.internal.util.tostring.*; +import org.gridgain.grid.kernal.processors.cache.*; import org.jetbrains.annotations.*; import javax.cache.configuration.*; @@ -186,6 +187,9 @@ public class GridCacheContext implements Externalizable { /** Default expiry policy. */ private ExpiryPolicy expiryPlc; + /** Cache weak query iterator holder. */ + private CacheWeakQueryIteratorsHolder> itHolder; + /** * Empty constructor required for {@link Externalizable}. */ @@ -300,6 +304,8 @@ public GridCacheContext( if (expiryPlc instanceof EternalExpiryPolicy) expiryPlc = null; + + itHolder = new CacheWeakQueryIteratorsHolder(log); } /** @@ -836,6 +842,13 @@ public GridCacheContinuousQueryManager continuousQueries() { return contQryMgr; } + /** + * @return Iterators Holder. + */ + public CacheWeakQueryIteratorsHolder> itHolder() { + return itHolder; + } + /** * @return Swap manager. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheGateway.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheGateway.java index ae97afee41be9..2de235a8ca755 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheGateway.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheGateway.java @@ -87,6 +87,8 @@ public void leave() { */ @Nullable public GridCacheProjectionImpl enter(@Nullable GridCacheProjectionImpl prj) { try { + ctx.itHolder().checkWeakQueue(); + GridCacheAdapter cache = ctx.cache(); GridCachePreloader preldr = cache != null ? cache.preloader() : null; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index b9265a5d10271..94ee239b6e338 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -820,27 +820,10 @@ else if (clazz.equals(Ignite.class)) /** {@inheritDoc} */ @Override public Iterator> iterator() { - // TODO IGNITE-1. GridCacheProjectionImpl prev = gate.enter(prj); try { - return F.iterator(delegate, new C1, Entry>() { - @Override public Entry apply(final CacheEntry e) { - return new Entry() { - @Override public K getKey() { - return e.getKey(); - } - - @Override public V getValue() { - return e.getValue(); - } - - @Override public T unwrap(Class clazz) { - throw new IllegalArgumentException(); - } - }; - } - }, false); + return ((GridCacheAdapter)delegate).igniteIterator(prj); } finally { gate.leave(prev); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheSetImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheSetImpl.java index 80151ec360fcf..b84d957a68902 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheSetImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheSetImpl.java @@ -30,6 +30,7 @@ import org.apache.ignite.internal.util.lang.*; import org.apache.ignite.internal.util.typedef.*; import org.apache.ignite.internal.util.typedef.internal.*; +import org.gridgain.grid.kernal.processors.cache.*; import org.jdk8.backport.*; import org.jetbrains.annotations.*; @@ -53,9 +54,6 @@ public class GridCacheSetImpl extends AbstractCollection implements CacheS /** Cache. */ private final GridCache cache; - /** Logger. */ - private final IgniteLogger log; - /** Set name. */ private final String name; @@ -71,12 +69,6 @@ public class GridCacheSetImpl extends AbstractCollection implements CacheS /** Removed flag. */ private volatile boolean rmvd; - /** Iterators weak references queue. */ - private final ReferenceQueue> itRefQueue = new ReferenceQueue<>(); - - /** Iterators futures. */ - private final Map>, CacheQueryFuture> itFuts = new ConcurrentHashMap8<>(); - /** * @param ctx Cache context. * @param name Set name. @@ -91,8 +83,6 @@ public GridCacheSetImpl(GridCacheContext ctx, String name, GridCacheSetHeader hd cache = ctx.cache(); - log = ctx.logger(GridCacheSetImpl.class); - hdrPart = ctx.affinity().partition(new GridCacheSetHeaderKey(name)); } @@ -348,16 +338,21 @@ private GridCloseableIterator iterator0() { qry.projection(ctx.grid().forNodes(nodes)); - CacheQueryFuture fut = qry.execute(); + CacheQueryFuture> fut = qry.execute(); - SetIterator it = new SetIterator<>(fut); + CacheWeakQueryIteratorsHolder.WeakQueryFutureIterator it = + ctx.itHolder().iterator(fut, new CacheIteratorConverter>() { + @Override protected T convert(Map.Entry e) { + return e.getKey(); + } - itFuts.put(it.weakReference(), fut); + @Override protected void remove(T item) { + GridCacheSetImpl.this.remove(item); + } + }); if (rmvd) { - itFuts.remove(it.weakReference()); - - it.close(); + ctx.itHolder().removeIterator(it); checkRemoved(); } @@ -443,18 +438,8 @@ void removed(boolean rmvd) { this.rmvd = rmvd; - if (rmvd) { - for (CacheQueryFuture fut : itFuts.values()) { - try { - fut.cancel(); - } - catch (IgniteCheckedException e) { - log.error("Failed to close iterator.", e); - } - } - - itFuts.clear(); - } + if (rmvd) + ctx.itHolder().clearQueries(); } /** @@ -465,30 +450,11 @@ private void checkRemoved() { throw new CacheDataStructureRemovedRuntimeException("Set has been removed from cache: " + this); } - /** - * Closes unreachable iterators. - */ - private void checkWeakQueue() { - for (Reference> itRef = itRefQueue.poll(); itRef != null; itRef = itRefQueue.poll()) { - try { - WeakReference> weakRef = (WeakReference>)itRef; - - CacheQueryFuture fut = itFuts.remove(weakRef); - - if (fut != null) - fut.cancel(); - } - catch (IgniteCheckedException e) { - log.error("Failed to close iterator.", e); - } - } - } - /** * Checks if set was removed and handles iterators weak reference queue. */ private void onAccess() { - checkWeakQueue(); + ctx.itHolder().checkWeakQueue(); checkRemoved(); } @@ -520,116 +486,6 @@ private GridCacheSetItemKey itemKey(Object item) { return S.toString(GridCacheSetImpl.class, this); } - /** - * - */ - private class SetIterator extends GridCloseableIteratorAdapter { - /** */ - private static final long serialVersionUID = -1460570789166994846L; - - /** Query future. */ - private final CacheQueryFuture fut; - - /** Init flag. */ - private boolean init; - - /** Next item. */ - private T next; - - /** Current item. */ - private T cur; - - /** Weak reference. */ - private final WeakReference> weakRef; - - /** - * @param fut Query future. - */ - private SetIterator(CacheQueryFuture fut) { - this.fut = fut; - - weakRef = new WeakReference>(this, itRefQueue); - } - - /** {@inheritDoc} */ - @Override protected T onNext() throws IgniteCheckedException { - init(); - - if (next == null) { - clearWeakReference(); - - throw new NoSuchElementException(); - } - - cur = next; - - Map.Entry e = (Map.Entry)fut.next(); - - next = e != null ? (T)e.getKey() : null; - - if (next == null) - clearWeakReference(); - - return cur; - } - - /** {@inheritDoc} */ - @Override protected boolean onHasNext() throws IgniteCheckedException { - init(); - - boolean hasNext = next != null; - - if (!hasNext) - clearWeakReference(); - - return hasNext; - } - - /** {@inheritDoc} */ - @Override protected void onClose() throws IgniteCheckedException { - fut.cancel(); - - clearWeakReference(); - } - - /** {@inheritDoc} */ - @Override protected void onRemove() throws IgniteCheckedException { - if (cur == null) - throw new NoSuchElementException(); - - GridCacheSetImpl.this.remove(cur); - } - - /** - * @throws IgniteCheckedException If failed. - */ - private void init() throws IgniteCheckedException { - if (!init) { - Map.Entry e = (Map.Entry)fut.next(); - - next = e != null ? (T)e.getKey() : null; - - init = true; - } - } - - /** - * @return Iterator weak reference. - */ - WeakReference> weakReference() { - return weakRef; - } - - /** - * Clears weak reference. - */ - private void clearWeakReference() { - weakRef.clear(); // Do not need to enqueue. - - itFuts.remove(weakRef); - } - } - /** * */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java index f65017a3bb212..d35d2153b1e98 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java @@ -65,13 +65,13 @@ public class GridCacheQueryAdapter implements CacheQuery { private volatile GridCacheQueryMetricsAdapter metrics; /** */ - private volatile int pageSize; + private volatile int pageSize = DFLT_PAGE_SIZE; /** */ private volatile long timeout; /** */ - private volatile boolean keepAll; + private volatile boolean keepAll = true; /** */ private volatile boolean incBackups; @@ -123,13 +123,6 @@ public GridCacheQueryAdapter(GridCacheContext cctx, log = cctx.logger(getClass()); - pageSize = DFLT_PAGE_SIZE; - timeout = 0; - keepAll = true; - incBackups = false; - dedup = false; - prj = null; - metrics = new GridCacheQueryMetricsAdapter(); } @@ -419,6 +412,9 @@ private CacheQueryFuture execute(@Nullable IgniteReducer rmtReducer cctx.checkSecurity(GridSecurityPermission.CACHE_READ); + if (nodes.isEmpty()) + return new GridCacheQueryErrorFuture<>(cctx.kernalContext(), new ClusterGroupEmptyException()); + if (log.isDebugEnabled()) log.debug("Executing query [query=" + this + ", nodes=" + nodes + ']'); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryFutureAdapter.java index 94c85b422ada4..4202c99d9d8bc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryFutureAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryFutureAdapter.java @@ -58,7 +58,7 @@ public abstract class GridCacheQueryFutureAdapter extends GridFutureAda protected final GridCacheQueryBean qry; /** Set of received keys used to deduplicate query result set. */ - private final Collection keys = new HashSet<>(); + private final Collection keys; /** */ private final Queue> queue = new LinkedList<>(); @@ -92,6 +92,7 @@ public abstract class GridCacheQueryFutureAdapter extends GridFutureAda */ protected GridCacheQueryFutureAdapter() { qry = null; + keys = null; } /** @@ -121,6 +122,8 @@ protected GridCacheQueryFutureAdapter(GridCacheContext cctx, GridCacheQuer cctx.time().addTimeoutObject(this); } + + keys = qry.query().enableDedup() ? new HashSet() : null; } /** @@ -335,7 +338,7 @@ private Collection dedupIfRequired(Collection col) { if (!qry.query().enableDedup()) return col; - Collection dedupCol = new LinkedList<>(); + Collection dedupCol = new ArrayList<>(col.size()); synchronized (mux) { for (Object o : col) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java index 5a0df9b26040f..ae5fe77c8c5d3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java @@ -756,8 +756,8 @@ private GridCloseableIterator> scanIterator(final GridCacheQ GridIterator> heapIt = new GridIteratorAdapter>() { private IgniteBiTuple next; - private Iterator iter = prj.keySet().iterator(); - + private Iterator iter = qry.includeBackups() || cctx.isReplicated() ? + prj.keySet().iterator() : prj.primaryKeySet().iterator(); { advance(); } diff --git a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheWeakQueryIteratorsHolder.java b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheWeakQueryIteratorsHolder.java index f955695e2c2d3..eea72fe8443a9 100644 --- a/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheWeakQueryIteratorsHolder.java +++ b/modules/core/src/main/java/org/gridgain/grid/kernal/processors/cache/CacheWeakQueryIteratorsHolder.java @@ -18,9 +18,9 @@ package org.gridgain.grid.kernal.processors.cache; import org.apache.ignite.*; -import org.gridgain.grid.cache.query.*; -import org.gridgain.grid.util.*; -import org.gridgain.grid.util.typedef.internal.*; +import org.apache.ignite.cache.query.*; +import org.apache.ignite.internal.util.*; +import org.apache.ignite.internal.util.typedef.internal.*; import org.jdk8.backport.*; import java.lang.ref.*; @@ -34,7 +34,7 @@ public class CacheWeakQueryIteratorsHolder { private final ReferenceQueue refQueue = new ReferenceQueue<>(); /** Iterators futures. */ - private final Map, GridCacheQueryFuture> futs = + private final Map, CacheQueryFuture> futs = new ConcurrentHashMap8<>(); /** Logger. */ @@ -53,10 +53,10 @@ public CacheWeakQueryIteratorsHolder(IgniteLogger log) { * @param Type for the iterator. * @return Iterator over the cache. */ - public WeakQueryFutureIterator iterator(GridCacheQueryFuture fut, CacheIteratorConverter convert) { + public WeakQueryFutureIterator iterator(CacheQueryFuture fut, CacheIteratorConverter convert) { WeakQueryFutureIterator it = new WeakQueryFutureIterator(fut, convert); - GridCacheQueryFuture old = futs.put(it.weakReference(), fut); + CacheQueryFuture old = futs.put(it.weakReference(), fut); assert old == null; @@ -83,7 +83,7 @@ public void checkWeakQueue() { try { WeakReference weakRef = (WeakReference)itRef; - GridCacheQueryFuture fut = futs.remove(weakRef); + CacheQueryFuture fut = futs.remove(weakRef); if (fut != null) fut.cancel(); @@ -98,7 +98,7 @@ public void checkWeakQueue() { * Cancel all cache queries. */ public void clearQueries(){ - for (GridCacheQueryFuture fut : futs.values()) { + for (CacheQueryFuture fut : futs.values()) { try { fut.cancel(); } @@ -112,13 +112,13 @@ public void clearQueries(){ /** - * Iterator based of {@link GridCacheQueryFuture}. + * Iterator based of {@link CacheQueryFuture}. * * @param Type for iterator. */ public class WeakQueryFutureIterator extends GridCloseableIteratorAdapter { /** Query future. */ - private final GridCacheQueryFuture fut; + private final CacheQueryFuture fut; /** Weak reference. */ private final WeakReference> weakRef; @@ -137,7 +137,7 @@ public class WeakQueryFutureIterator extends GridCloseableIteratorAdapter /** * @param fut GridCacheQueryFuture to iterate. */ - WeakQueryFutureIterator(GridCacheQueryFuture fut, CacheIteratorConverter convert) { + WeakQueryFutureIterator(CacheQueryFuture fut, CacheIteratorConverter convert) { this.fut = fut; this.weakRef = new WeakReference<>(this, refQueue); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java index 7c6af953c4c5c..b0b0a5a685625 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java @@ -18,12 +18,14 @@ package org.apache.ignite.internal.processors.cache; import com.google.common.collect.*; +import junit.framework.*; import org.apache.ignite.*; import org.apache.ignite.cache.*; import org.apache.ignite.cluster.*; import org.apache.ignite.configuration.*; import org.apache.ignite.events.*; import org.apache.ignite.internal.*; +import org.apache.ignite.internal.processors.cache.query.*; import org.apache.ignite.lang.*; import org.apache.ignite.transactions.*; import org.apache.ignite.spi.swapspace.inmemory.*; @@ -33,6 +35,7 @@ import org.apache.ignite.testframework.*; import org.jetbrains.annotations.*; +import javax.cache.*; import javax.cache.expiry.*; import javax.cache.processor.*; import java.util.*; @@ -5087,4 +5090,179 @@ protected List primaryKeysForCache(IgniteCache cache, i return primaryKeysForCache(prj, cnt); } + + /** + * @throws Exception If failed. + */ + public void testIgniteCacheIterator() throws Exception { + IgniteCache cache = jcache(0); + + assertFalse(cache.iterator().hasNext()); + + final int SIZE = 20000; + + Map entries = new HashMap<>(); + + for (int i = 0; i < SIZE; ++i) { + cache.put(Integer.toString(i), i); + + entries.put(Integer.toString(i), i); + } + + checkIteratorHasNext(); + + checkIteratorCache(entries); + + checkIteratorRemove(cache, entries); + + checkIteratorEmpty(cache); + } + + /** + * If hasNext() is called repeatedly, it should return the same result. + */ + private void checkIteratorHasNext() { + Iterator> iter = jcache(0).iterator(); + + assertEquals(iter.hasNext(), iter.hasNext()); + + while (iter.hasNext()) + iter.next(); + + assertFalse(iter.hasNext()); + } + + /** + * @param cache Cache. + * @param entries Expected entries in the cache. + */ + private void checkIteratorRemove(IgniteCache cache, Map entries) { + // Check that we can remove element. + String rmvKey = Integer.toString(5); + + removeCacheIterator(cache, rmvKey); + + entries.remove(rmvKey); + + assertFalse(cache.containsKey(rmvKey)); + assertNull(cache.get(rmvKey)); + + checkIteratorCache(entries); + + // Check that we cannot call Iterator.remove() without next(). + final Iterator> iter = jcache(0).iterator(); + + assertTrue(iter.hasNext()); + + iter.next(); + + iter.remove(); + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Void call() throws Exception { + iter.remove(); + + return null; + } + }, IllegalStateException.class, null); + } + + /** + * @param cache Cache. + * @param key Key to remove. + */ + private void removeCacheIterator(IgniteCache cache, String key) { + Iterator> iter = cache.iterator(); + + int delCnt = 0; + + while (iter.hasNext()) { + Cache.Entry cur = iter.next(); + + if (cur.getKey().equals(key)) { + iter.remove(); + + delCnt++; + } + } + + assertEquals(1, delCnt); + } + + /** + * @param entries Expected entries in the cache. + */ + private void checkIteratorCache(Map entries) { + for (int i = 0; i < gridCount(); ++i) + checkIteratorCache(jcache(i), entries); + } + + /** + * @param cache Cache. + * @param entries Expected entries in the cache. + */ + private void checkIteratorCache(IgniteCache cache, Map entries) { + Iterator> iter = cache.iterator(); + + int cnt = 0; + + while (iter.hasNext()) { + Cache.Entry cur = iter.next(); + + assertTrue(entries.containsKey(cur.getKey())); + assertEquals(entries.get(cur.getKey()), cur.getValue()); + + cnt++; + } + + assertEquals(entries.size(), cnt); + } + + /** + * Checks iterators are cleared. + */ + private void checkIteratorsCleared() { + for (int j = 0; j < gridCount(); j++) { + + GridCacheQueryManager queries = context(j).queries(); + + Map map = GridTestUtils.getFieldValue(queries, GridCacheQueryManager.class, "qryIters"); + + for (Object obj : map.values()) + assertEquals("Iterators not removed for grid " + j, 0, ((Map) obj).size()); + } + } + + /** + * Checks iterators are cleared after using. + */ + private void checkIteratorEmpty(IgniteCache cache) throws InterruptedException, InterruptedException { + int cnt = 5; + + for (int i = 0; i < cnt; ++i) { + Iterator> iter = cache.iterator(); + + iter.next(); + + assert iter.hasNext(); + } + + System.gc(); + + for (int i = 0; i < 10; i++) { + try { + cache.size(); // Trigger weak queue poll. + + checkIteratorsCleared(); + } + catch (AssertionFailedError e) { + if (i == 9) + throw e; + + log.info("Set iterators not cleared, will wait"); + + Thread.sleep(500); + } + } + } } From e9ea63e23ef25c93978b7e1825aecde566233f29 Mon Sep 17 00:00:00 2001 From: ivasilinets Date: Fri, 23 Jan 2015 19:10:59 +0300 Subject: [PATCH 33/36] #IGNITE-66: merge --- examples/pom-standalone.xml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/examples/pom-standalone.xml b/examples/pom-standalone.xml index 78c07366aae76..dc0f9cab1372a 100644 --- a/examples/pom-standalone.xml +++ b/examples/pom-standalone.xml @@ -32,50 +32,50 @@ src/main/java - org.gridgain - gridgain-examples + org.apache.ignite + ignite-examples ${ignite.version} - org.gridgain - gridgain-core + org.apache.ignite + ignite-core ${ignite.version} - org.gridgain - gridgain-hibernate + org.apache.ignite + ignite-hibernate ${ignite.version} - org.gridgain - gridgain-scalar + org.apache.ignite + ignite-scalar ${ignite.version} - org.gridgain - gridgain-spring + org.apache.ignite + ignite-spring ${ignite.version} - org.gridgain - gridgain-log4j + org.apache.ignite + ignite-log4j ${ignite.version} - org.gridgain - gridgain-indexing + org.apache.ignite + ignite-indexing ${ignite.version} - org.gridgain - gridgain-schedule + org.apache.ignite + ignite-schedule ${ignite.version} From df849b18d05f88a0075363cf13436e7715edde3c Mon Sep 17 00:00:00 2001 From: Dmitiry Setrakyan Date: Fri, 23 Jan 2015 12:22:33 -0800 Subject: [PATCH 34/36] Style fix. --- .../src/main/java/org/apache/ignite/IgniteCachingProvider.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteCachingProvider.java b/modules/core/src/main/java/org/apache/ignite/IgniteCachingProvider.java index 0560dad679737..0a60a558f6858 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteCachingProvider.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteCachingProvider.java @@ -42,11 +42,12 @@ public class IgniteCachingProvider implements CachingProvider { try { URL dfltCfgURL = U.resolveGridGainUrl(GridGainEx.DFLT_CFG); + if (dfltCfgURL != null) uri = dfltCfgURL.toURI(); } catch (URISyntaxException ignored) { - + // No-op. } if (uri == null) From 833755acbdfd94fd4a198a334341c78dccc3f2ca Mon Sep 17 00:00:00 2001 From: Dmitiry Setrakyan Date: Sat, 24 Jan 2015 11:09:48 -0800 Subject: [PATCH 35/36] # sprint-1 Notice file. --- NOTICE.txt | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100755 NOTICE.txt diff --git a/NOTICE.txt b/NOTICE.txt new file mode 100755 index 0000000000000..cda478660b84a --- /dev/null +++ b/NOTICE.txt @@ -0,0 +1,46 @@ +Apache Ignite (incubating) +Copyright [2014-2015] The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This software has a dependency on Java port of a concurrent trie hash +map implementation from the Scala collections library, licensed +under Apache 2.0: https://github.com/romix/java-concurrent-hash-trie-map + +This software has a dependency on JetBrains Annotations licensed under +Apache 2.0 license. + +This software has a dependency on Spring software licensed under Apache 2.0 +license: https://spring.io/ + +This software has a dependency on AWS Java SDK licensed under Apache 2.0 +license: http://aws.amazon.com/sdk-for-java/ + +This software has a dependency on Jetty Servlet Engine licensed +under Apache 2.0: http://eclipse.org/jetty/ + +This software has a dependency on H2 database licensed under +Eclipse Public License: http://www.h2database.com/ + +This software has a dependency on AspectJ 1.7 licensed under +Eclipse Public License: https://eclipse.org/aspectj/ + +This software has a dependency on SLF4J licensed under MIT license: +http://www.slf4j.org/ + +This software has a dependency on Persistent Collections library, licensed +under MIT license: https://github.com/blackdrag/pcollections + +This software has a dependency on Snaptree library licensed under +New BSD License: https://github.com/nbronson/snaptree/ + +This software has a dependency on Scala licensed under New BSD license: +http://www.scala-lang.org/ + +This software has a dependency on Java Secure Channel library licensed +under New BSD license: http://www.jcraft.com/jsch/ + + + + From 89704638a2a5c3df8ed4257c400b55fa8ca1c8a0 Mon Sep 17 00:00:00 2001 From: Dmitiry Setrakyan Date: Sat, 24 Jan 2015 18:13:59 -0800 Subject: [PATCH 36/36] # sprint-1 Notice file. --- NOTICE.txt | 70 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 50 insertions(+), 20 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index cda478660b84a..dbf2072f0bb3b 100755 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,6 +1,9 @@ Apache Ignite (incubating) Copyright [2014-2015] The Apache Software Foundation +Source Code Dependencies +======================== + This product includes software developed at The Apache Software Foundation (http://www.apache.org/). @@ -11,36 +14,63 @@ under Apache 2.0: https://github.com/romix/java-concurrent-hash-trie-map This software has a dependency on JetBrains Annotations licensed under Apache 2.0 license. -This software has a dependency on Spring software licensed under Apache 2.0 -license: https://spring.io/ +This software has a dependency on Persistent Collections library, +licensed under MIT license: https://github.com/blackdrag/pcollections + + + +Optional Runtime Dependencies +============================= + +Apache 2.0 +---------- +This software has an optional runtime dependency on Spring software +licensed under Apache 2.0 license: https://spring.io/ + +This software has an optional runtime dependency on AWS Java SDK +licensed under Apache 2.0 license: http://aws.amazon.com/sdk-for-java/ + +This software has an optional runtime dependency on Jetty Servlet +Engine licensed under Apache 2.0: http://eclipse.org/jetty/ + -This software has a dependency on AWS Java SDK licensed under Apache 2.0 -license: http://aws.amazon.com/sdk-for-java/ +Eclipse Public License +---------------------- +This software has an optional runtime dependency on H2 database +licensed under Eclipse Public License: http://www.h2database.com/ -This software has a dependency on Jetty Servlet Engine licensed -under Apache 2.0: http://eclipse.org/jetty/ +This software has an optional runtime dependency on AspectJ 1.7 +licensed under Eclipse Public License: https://eclipse.org/aspectj/ -This software has a dependency on H2 database licensed under -Eclipse Public License: http://www.h2database.com/ -This software has a dependency on AspectJ 1.7 licensed under -Eclipse Public License: https://eclipse.org/aspectj/ +MIT License +----------- +This software has an optional runtime dependency on SLF4J licensed +under MIT license: http://www.slf4j.org/ -This software has a dependency on SLF4J licensed under MIT license: -http://www.slf4j.org/ -This software has a dependency on Persistent Collections library, licensed -under MIT license: https://github.com/blackdrag/pcollections +New BSD License +--------------- +This software has an optional runtime dependency on Snaptree library +licensed under New BSD License: https://github.com/nbronson/snaptree/ -This software has a dependency on Snaptree library licensed under -New BSD License: https://github.com/nbronson/snaptree/ +This software has an optional runtime dependency on Scala licensed +under New BSD license: http://www.scala-lang.org/ -This software has a dependency on Scala licensed under New BSD license: -http://www.scala-lang.org/ +This software has an optional runtime dependency on Java Secure Channel +library licensed under New BSD license: http://www.jcraft.com/jsch/ -This software has a dependency on Java Secure Channel library licensed -under New BSD license: http://www.jcraft.com/jsch/ +LGPL +---- +This software has an optional runtime dependency on Hibernate ORM +licensed under LGPL: http://hibernate.org/orm/ +This software has an optional runtime dependency on edtFTPj library +licensed under LGPL: http://enterprisedt.com/products/edtftpj/ +This software has an optional runtime dependency on Cron4j library +licensed under LGPL: http://www.sauronsoftware.it/projects/cron4j/ +This software has an optional runtime dependency on JTS Topology Suite +licensed under LGPL: http://www.vividsolutions.com/jts/JTSHome.htm \ No newline at end of file