From 219f28948d22a36f9292ae34b02e18171486d3cc Mon Sep 17 00:00:00 2001 From: oleg-ostanin Date: Mon, 17 Jul 2017 16:29:16 +0300 Subject: [PATCH 001/547] removed excluding ML sources from assembly file - Fixes #2310. Signed-off-by: Alexey Goncharuk --- assembly/release-fabric-base.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/assembly/release-fabric-base.xml b/assembly/release-fabric-base.xml index 7484dfa9505c2..5007785bb2a58 100644 --- a/assembly/release-fabric-base.xml +++ b/assembly/release-fabric-base.xml @@ -239,7 +239,6 @@ **/package.html src/test/** - src/main/ml/** From 9cd966a702faff32f6efc7fe9e05294b52ecfffc Mon Sep 17 00:00:00 2001 From: oleg-ostanin Date: Mon, 17 Jul 2017 16:30:04 +0300 Subject: [PATCH 002/547] removed excluding ML sources from assembly file - Fixes #2310. Signed-off-by: Alexey Goncharuk --- assembly/release-fabric-base.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/assembly/release-fabric-base.xml b/assembly/release-fabric-base.xml index 7484dfa9505c2..5007785bb2a58 100644 --- a/assembly/release-fabric-base.xml +++ b/assembly/release-fabric-base.xml @@ -239,7 +239,6 @@ **/package.html src/test/** - src/main/ml/** From e88fcd8d228e2f15187ff05218f3afb242c65fed Mon Sep 17 00:00:00 2001 From: devozerov Date: Mon, 17 Jul 2017 17:13:16 +0300 Subject: [PATCH 003/547] Revert "removed excluding ML sources from assembly file - Fixes #2310." This reverts commit 219f28948d22a36f9292ae34b02e18171486d3cc. --- assembly/release-fabric-base.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/assembly/release-fabric-base.xml b/assembly/release-fabric-base.xml index 5007785bb2a58..7484dfa9505c2 100644 --- a/assembly/release-fabric-base.xml +++ b/assembly/release-fabric-base.xml @@ -239,6 +239,7 @@ **/package.html src/test/** + src/main/ml/** From 70d0f9918c708cb117e69163cc7b7c119c9a693c Mon Sep 17 00:00:00 2001 From: Dmitriy Shabalin Date: Thu, 20 Jul 2017 15:08:20 +0700 Subject: [PATCH 004/547] IGNITE-4728 Web Console: Saved last succeeded state and redirect to it on reload. --- modules/web-console/frontend/app/app.js | 14 ++++++++++++++ .../frontend/app/modules/states/errors.state.js | 6 ++++-- .../frontend/app/modules/states/signin.state.js | 10 +++++++++- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js index c707810424769..dc5c6e9729567 100644 --- a/modules/web-console/frontend/app/app.js +++ b/modules/web-console/frontend/app/app.js @@ -289,6 +289,20 @@ angular $root.$on('$stateChangeStart', () => { _.forEach(angular.element('.modal'), (m) => angular.element(m).scope().$hide()); }); + + if (!$root.IgniteDemoMode) { + $root.$on('$stateChangeSuccess', (event, {name, unsaved}, params) => { + try { + if (unsaved) + localStorage.removeItem('lastStateChangeSuccess'); + else + localStorage.setItem('lastStateChangeSuccess', JSON.stringify({name, params})); + } + catch (ignored) { + // No-op. + } + }); + } }]) .run(['$rootScope', '$http', '$state', 'IgniteMessages', 'User', 'IgniteNotebookData', ($root, $http, $state, Messages, User, Notebook) => { // eslint-disable-line no-shadow diff --git a/modules/web-console/frontend/app/modules/states/errors.state.js b/modules/web-console/frontend/app/modules/states/errors.state.js index e816ff80224ae..e3d4d415811ee 100644 --- a/modules/web-console/frontend/app/modules/states/errors.state.js +++ b/modules/web-console/frontend/app/modules/states/errors.state.js @@ -31,13 +31,15 @@ angular templateUrl: templateNotFoundPage, metaTags: { title: 'Page not found' - } + }, + unsaved: true }) .state('403', { url: '/403', templateUrl: templateNotAuthorizedPage, metaTags: { title: 'Not authorized' - } + }, + unsaved: true }); }]); diff --git a/modules/web-console/frontend/app/modules/states/signin.state.js b/modules/web-console/frontend/app/modules/states/signin.state.js index 5155bdec6c304..b7be51d91b4b2 100644 --- a/modules/web-console/frontend/app/modules/states/signin.state.js +++ b/modules/web-console/frontend/app/modules/states/signin.state.js @@ -33,7 +33,15 @@ angular resolve: { user: ['$state', 'User', ($state, User) => { return User.read() - .then(() => $state.go('base.configuration.tabs')) + .then(() => { + try { + const {name, params} = JSON.parse(localStorage.getItem('lastStateChangeSuccess')); + + $state.go(name, params); + } catch (ignored) { + $state.go('base.configuration.tabs'); + } + }) .catch(() => {}); }] }, From 02a1bdca57ce6af7fe7636b0a9f99048c89b88b6 Mon Sep 17 00:00:00 2001 From: Andrey Novikov Date: Thu, 20 Jul 2017 15:47:49 +0700 Subject: [PATCH 005/547] IGNITE-5788 Web Console: Fixed dependencies for maven project with c3p0. --- .../web-console/frontend/app/data/pom-dependencies.json | 5 ++++- .../app/modules/configuration/generator/Maven.service.js | 9 +++------ modules/web-console/frontend/webpack/webpack.common.js | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/web-console/frontend/app/data/pom-dependencies.json b/modules/web-console/frontend/app/data/pom-dependencies.json index 945e3f55f0886..8d3fa81fa4a4e 100644 --- a/modules/web-console/frontend/app/data/pom-dependencies.json +++ b/modules/web-console/frontend/app/data/pom-dependencies.json @@ -11,7 +11,10 @@ "HadoopIgfsJcl": {"artifactId": "ignite-hadoop"}, "SLF4J": {"artifactId": "ignite-slf4j"}, - "Generic": {"groupId": "com.mchange", "artifactId": "c3p0", "version": "0.9.5.2"}, + "Generic": [ + {"groupId": "com.mchange", "artifactId": "c3p0", "version": "0.9.5.2"}, + {"groupId": "com.mchange", "artifactId": "mchange-commons-java", "version": "0.2.11"} + ], "MySQL": {"groupId": "mysql", "artifactId": "mysql-connector-java", "version": "5.1.40"}, "PostgreSQL": {"groupId": "org.postgresql", "artifactId": "postgresql", "version": "9.4.1212.jre7"}, "H2": {"groupId": "com.h2database", "artifactId": "h2", "version": [ diff --git a/modules/web-console/frontend/app/modules/configuration/generator/Maven.service.js b/modules/web-console/frontend/app/modules/configuration/generator/Maven.service.js index 81d7d10541600..abbada93bf397 100644 --- a/modules/web-console/frontend/app/modules/configuration/generator/Maven.service.js +++ b/modules/web-console/frontend/app/modules/configuration/generator/Maven.service.js @@ -47,12 +47,9 @@ export default class IgniteMavenGenerator { return _.isArray(version) ? _.find(version, (v) => versionService.since(igniteVer, v.range)).version : version; }; - if (!_.has(POM_DEPENDENCIES, key)) - return; - - const {groupId, artifactId, version, jar} = POM_DEPENDENCIES[key]; - - this.addDependency(deps, groupId || 'org.apache.ignite', artifactId, extractVersion(version) || dfltVer, jar); + _.forEach(POM_DEPENDENCIES[key], ({groupId, artifactId, version, jar}) => { + this.addDependency(deps, groupId || 'org.apache.ignite', artifactId, extractVersion(version) || dfltVer, jar); + }); } addResource(sb, dir, exclude) { diff --git a/modules/web-console/frontend/webpack/webpack.common.js b/modules/web-console/frontend/webpack/webpack.common.js index a303d6e5d30f5..5a3763e9929f8 100644 --- a/modules/web-console/frontend/webpack/webpack.common.js +++ b/modules/web-console/frontend/webpack/webpack.common.js @@ -138,7 +138,7 @@ export default { }, { test: /\.(jpe?g|png|gif)$/i, - loader: 'file?name=assets/images/[name]_[hash].[ext]' + loader: 'file?name=assets/images/[name].[hash].[ext]' }, { test: require.resolve('jquery'), @@ -178,7 +178,7 @@ export default { new HtmlWebpackPlugin({ template: './views/index.pug' }), - new ExtractTextPlugin({filename: 'assets/css/[name].css', allChunks: true}), + new ExtractTextPlugin({filename: 'assets/css/[name].[hash].css', allChunks: true}), new CopyWebpackPlugin([ { context: 'public', from: '**/*.png' }, { context: 'public', from: '**/*.svg' }, From bdaeecca96798539c4baebe30aa10b33d832df10 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Thu, 20 Jul 2017 14:58:09 +0300 Subject: [PATCH 006/547] Assembly procedure fix --- DEVNOTES.txt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/DEVNOTES.txt b/DEVNOTES.txt index 8a689b8cefab8..340153ed23203 100644 --- a/DEVNOTES.txt +++ b/DEVNOTES.txt @@ -2,14 +2,14 @@ Ignite Fabric Maven Build Instructions ====================================== 1) Optional: build Apache Ignite.NET as described at modules/platforms/dotnet/DEVNOTES.txt. -2) Compile and package: +2) Compile and install: - mvn clean package -Pall-java,all-scala,licenses -DskipTests + mvn clean install -Pall-java,all-scala,licenses -DskipTests or if you have built Apache Ignite.NET on the first step use following command: (Note that 'doxygen' should be installed before running this command.) - mvn clean package -Pall-java,all-scala,licenses -DskipTests -DclientDocs + mvn clean install -Pall-java,all-scala,licenses -DskipTests -DclientDocs 3) Javadoc generation (optional): @@ -25,14 +25,14 @@ Ignite Fabric with LGPL Maven Build Instructions ====================================== 1) Optional: build Apache Ignite.NET as described at modules/platforms/dotnet/DEVNOTES.txt. -2) Compile and package: +2) Compile and install: - mvn clean package -Pall-java,all-scala,licenses -DskipTests + mvn clean install -Pall-java,all-scala,licenses -DskipTests or if you have built Apache Ignite.NET on the first step use following command: (Note that 'doxygen' should be installed before running this command.) - mvn clean package -Pall-java,all-scala,licenses -DskipTests -DclientDocs + mvn clean install -Pall-java,all-scala,licenses -DskipTests -DclientDocs 3) Javadoc generation with LGPL (optional): @@ -46,15 +46,15 @@ Look for apache-ignite-fabric-lgpl--bin.zip in ./target/bin directory. Ignite Hadoop Accelerator Maven Build Instructions ============================================ -1) Compile and package: +1) Compile and install: - mvn clean package -Pall-java,all-scala,licenses -DskipTests + mvn clean install -Pall-java,all-scala,licenses -DskipTests Use 'hadoop.version' parameter to build Ignite against a specific Hadoop version. Use 'spark.version' parameter to build ignite-spark module for a specific Spark version. Version should be >= 2.0.0. For example: - mvn clean package -Pall-java,all-scala,licenses -DskipTests -Dhadoop.version=2.4.2 -Dspark.version=2.1.1 + mvn clean install -Pall-java,all-scala,licenses -DskipTests -Dhadoop.version=2.4.2 -Dspark.version=2.1.1 2) Assembly Hadoop Accelerator: mvn initialize -Prelease -Dignite.edition=hadoop From 0d2b989d2be62533a36061940497a734463b5f10 Mon Sep 17 00:00:00 2001 From: Yury Babak Date: Fri, 21 Jul 2017 15:28:21 +0300 Subject: [PATCH 007/547] IGNITE-5791 Block matrix introduction This closes #2326 --- .../ignite/ml/math/DistanceMeasure.java | 2 +- .../ignite/ml/math/EuclideanDistance.java | 3 +- .../decompositions/EigenDecomposition.java | 2 +- .../ignite/ml/math/impls/CacheUtils.java | 198 ++++++-- .../ml/math/impls/matrix/AbstractMatrix.java | 4 +- .../ml/math/impls/matrix/BlockEntry.java | 50 ++ .../ml/math/impls/matrix/CacheMatrix.java | 9 +- .../matrix/SparseBlockDistributedMatrix.java | 208 +++++++++ .../impls/matrix/SparseDistributedMatrix.java | 26 +- .../storage/matrix/BaseBlockMatrixKey.java | 41 ++ .../impls/storage/matrix/BlockMatrixKey.java | 144 ++++++ .../storage/matrix/BlockMatrixStorage.java | 435 ++++++++++++++++++ .../SparseLocalOnHeapVectorStorage.java | 4 +- .../ignite/ml/math/statistics/Variance.java | 1 + .../ml/math/statistics/package-info.java | 22 + .../apache/ignite/ml/math/util/MapUtil.java | 2 +- .../ignite/ml/math/util/package-info.java | 22 + .../org/apache/ignite/ml/package-info.java | 22 + .../ml/math/MathImplDistributedTestSuite.java | 2 + .../SparseDistributedBlockMatrixTest.java | 379 +++++++++++++++ .../matrix/SparseDistributedMatrixTest.java | 32 +- 21 files changed, 1528 insertions(+), 80 deletions(-) create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/BlockEntry.java create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseBlockDistributedMatrix.java create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BaseBlockMatrixKey.java create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixKey.java create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/package-info.java create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/math/util/package-info.java create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/package-info.java create mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java index 09be0c3258766..df235a76975a9 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java @@ -34,5 +34,5 @@ public interface DistanceMeasure extends Externalizable { * @return the distance between the two vectors * @throws CardinalityException if the array lengths differ. */ - double compute(Vector a, Vector b) throws CardinalityException; + public double compute(Vector a, Vector b) throws CardinalityException; } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java index 5f962ce1ec4ca..edc11dc98733d 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java @@ -30,8 +30,7 @@ public class EuclideanDistance implements DistanceMeasure { private static final long serialVersionUID = 1717556319784040040L; /** {@inheritDoc} */ - @Override - public double compute(Vector a, Vector b) + @Override public double compute(Vector a, Vector b) throws CardinalityException { return MatrixUtil.localCopyOf(a).minus(b).kNorm(2.0); } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/EigenDecomposition.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/EigenDecomposition.java index d0e91a56e841f..a5c92e68f8d41 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/EigenDecomposition.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/EigenDecomposition.java @@ -446,7 +446,7 @@ private void hqr2(Matrix h) { // Store roots isolated by balanc and compute matrix norm - double norm = h.foldMap(Functions.PLUS, Functions.ABS, 0.0); + double norm = h.foldMap(Functions.PLUS, Functions.ABS, 0.0d); // Outer loop over eigenvalue index diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java index 1bda5e6f2b3e1..369840b2bf2b7 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java @@ -39,11 +39,16 @@ import org.apache.ignite.ml.math.ValueMapper; import org.apache.ignite.ml.math.functions.IgniteBiFunction; import org.apache.ignite.ml.math.functions.IgniteConsumer; +import org.apache.ignite.ml.math.functions.IgniteDoubleFunction; import org.apache.ignite.ml.math.functions.IgniteFunction; -import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage; +import org.apache.ignite.ml.math.impls.matrix.BlockEntry; +import org.apache.ignite.ml.math.impls.storage.matrix.BlockMatrixKey; +import org.apache.ignite.internal.util.typedef.internal.A; /** * Distribution-related misc. support. + * + * TODO: IGNITE-5102, fix sparse key filters */ public class CacheUtils { /** @@ -127,19 +132,38 @@ public static double sum(String cacheName, KeyMapper keyMapper, ValueM * @param matrixUuid Matrix UUID. * @return Sum obtained using sparse logic. */ - public static double sparseSum(IgniteUuid matrixUuid) { - Collection subSums = fold(SparseDistributedMatrixStorage.ML_CACHE_NAME, (CacheEntry, Map> ce, Double acc) -> { - Cache.Entry, Map> entry = ce.entry(); - if (entry.getKey().get2().equals(matrixUuid)) { - Map map = entry.getValue(); + @SuppressWarnings("unchecked") + public static double sparseSum(IgniteUuid matrixUuid, String cacheName) { + A.notNull(matrixUuid, "matrixUuid"); + A.notNull(cacheName, "cacheName"); + + Collection subSums = fold(cacheName, (CacheEntry ce, Double acc) -> { + V v = ce.entry().getValue(); + + double sum = 0.0; - double sum = sum(map.values()); + if (v instanceof Map) { + Map map = (Map)v; - return acc == null ? sum : acc + sum; + sum = sum(map.values()); + } + else if (v instanceof BlockEntry) { + BlockEntry be = (BlockEntry)v; + + sum = be.sum(); } else - return acc; - }, key -> key.get2().equals(matrixUuid)); + throw new UnsupportedOperationException(); + + return acc == null ? sum : acc + sum; + }, key -> { + if (key instanceof BlockMatrixKey) + return ((BlockMatrixKey)key).matrixId().equals(matrixUuid); + else if (key instanceof IgniteBiTuple) + return ((IgniteBiTuple)key).get2().equals(matrixUuid); + else + throw new UnsupportedOperationException(); + }); return sum(subSums); } @@ -186,23 +210,42 @@ public static double min(String cacheName, KeyMapper keyMapper, ValueM * @param matrixUuid Matrix UUID. * @return Minimum value obtained using sparse logic. */ - public static double sparseMin(IgniteUuid matrixUuid) { - Collection mins = fold(SparseDistributedMatrixStorage.ML_CACHE_NAME, (CacheEntry, Map> ce, Double acc) -> { - Cache.Entry, Map> entry = ce.entry(); + @SuppressWarnings("unchecked") + public static double sparseMin(IgniteUuid matrixUuid, String cacheName) { + A.notNull(matrixUuid, "matrixUuid"); + A.notNull(cacheName, "cacheName"); - if (entry.getKey().get2().equals(matrixUuid)) { - Map map = entry.getValue(); + Collection mins = fold(cacheName, (CacheEntry ce, Double acc) -> { + V v = ce.entry().getValue(); - double min = Collections.min(map.values()); + double min; - if (acc == null) - return min; - else - return Math.min(acc, min); + if (v instanceof Map) { + Map map = (Map)v; + + min = Collections.min(map.values()); + } + else if (v instanceof BlockEntry) { + BlockEntry be = (BlockEntry)v; + + min = be.minValue(); } else - return acc; - }, key -> key.get2().equals(matrixUuid)); + throw new UnsupportedOperationException(); + + if (acc == null) + return min; + else + return Math.min(acc, min); + + }, key -> { + if (key instanceof BlockMatrixKey) + return ((BlockMatrixKey)key).matrixId().equals(matrixUuid); + else if (key instanceof IgniteBiTuple) + return ((IgniteBiTuple)key).get2().equals(matrixUuid); + else + throw new UnsupportedOperationException(); + }); return Collections.min(mins); } @@ -211,22 +254,42 @@ public static double sparseMin(IgniteUuid matrixUuid) { * @param matrixUuid Matrix UUID. * @return Maximum value obtained using sparse logic. */ - public static double sparseMax(IgniteUuid matrixUuid) { - Collection maxes = fold(SparseDistributedMatrixStorage.ML_CACHE_NAME, (CacheEntry, Map> ce, Double acc) -> { - Cache.Entry, Map> entry = ce.entry(); - if (entry.getKey().get2().equals(matrixUuid)) { - Map map = entry.getValue(); + @SuppressWarnings("unchecked") + public static double sparseMax(IgniteUuid matrixUuid, String cacheName) { + A.notNull(matrixUuid, "matrixUuid"); + A.notNull(cacheName, "cacheName"); + + Collection maxes = fold(cacheName, (CacheEntry ce, Double acc) -> { + V v = ce.entry().getValue(); - double max = Collections.max(map.values()); + double max; - if (acc == null) - return max; - else - return Math.max(acc, max); + if (v instanceof Map) { + Map map = (Map)v; + + max = Collections.max(map.values()); + } + else if (v instanceof BlockEntry) { + BlockEntry be = (BlockEntry)v; + + max = be.maxValue(); } else - return acc; - }, key -> key.get2().equals(matrixUuid)); + throw new UnsupportedOperationException(); + + if (acc == null) + return max; + else + return Math.max(acc, max); + + }, key -> { + if (key instanceof BlockMatrixKey) + return ((BlockMatrixKey)key).matrixId().equals(matrixUuid); + else if (key instanceof IgniteBiTuple) + return ((IgniteBiTuple)key).get2().equals(matrixUuid); + else + throw new UnsupportedOperationException(); + }); return Collections.max(maxes); } @@ -279,17 +342,41 @@ public static void map(String cacheName, KeyMapper keyMapper, ValueMap * @param matrixUuid Matrix UUID. * @param mapper Mapping {@link IgniteFunction}. */ - public static void sparseMap(IgniteUuid matrixUuid, IgniteFunction mapper) { - foreach(SparseDistributedMatrixStorage.ML_CACHE_NAME, (CacheEntry, Map> ce) -> { - IgniteBiTuple k = ce.entry().getKey(); + @SuppressWarnings("unchecked") + public static void sparseMap(IgniteUuid matrixUuid, IgniteDoubleFunction mapper, String cacheName) { + A.notNull(matrixUuid, "matrixUuid"); + A.notNull(cacheName, "cacheName"); + A.notNull(mapper, "mapper"); + + foreach(cacheName, (CacheEntry ce) -> { + K k = ce.entry().getKey(); + + V v = ce.entry().getValue(); - Map v = ce.entry().getValue(); + if (v instanceof Map) { + Map map = (Map)v; - for (Map.Entry e : v.entrySet()) - e.setValue(mapper.apply(e.getValue())); + for (Map.Entry e : (map.entrySet())) + e.setValue(mapper.apply(e.getValue())); + + } + else if (v instanceof BlockEntry) { + BlockEntry be = (BlockEntry)v; + + be.map(mapper); + } + else + throw new UnsupportedOperationException(); ce.cache().put(k, v); - }, key -> key.get2().equals(matrixUuid)); + }, key -> { + if (key instanceof BlockMatrixKey) + return ((BlockMatrixKey)key).matrixId().equals(matrixUuid); + else if (key instanceof IgniteBiTuple) + return ((IgniteBiTuple)key).get2().equals(matrixUuid); + else + throw new UnsupportedOperationException(); + }); } /** @@ -327,8 +414,7 @@ public static void foreach(String cacheName, IgniteConsumer entry : cache.query(new ScanQuery(part, - (k, v) -> affinity.mapPartitionToNode(p) == locNode && (keyFilter == null || keyFilter.apply(k))))) + for (Cache.Entry entry : cache.query(new ScanQuery(part, (k, v) -> affinity.mapPartitionToNode(p) == locNode && (keyFilter == null || keyFilter.apply(k))))) fun.accept(new CacheEntry<>(entry, cache)); } }); @@ -387,12 +473,34 @@ public static Collection fold(String cacheName, IgniteBiFunction A distributedFold(String cacheName, IgniteBiFunction, A, A> folder, IgnitePredicate keyFilter, BinaryOperator accumulator, A zeroVal) { return sparseFold(cacheName, folder, keyFilter, accumulator, zeroVal, null, null, 0, false); } + /** + * Sparse version of fold. This method also applicable to sparse zeroes. + * + * @param cacheName Cache name. + * @param folder Folder. + * @param keyFilter Key filter. + * @param accumulator Accumulator. + * @param zeroVal Zero value. + * @param defVal Def value. + * @param defKey Def key. + * @param defValCnt Def value count. + * @param isNilpotent Is nilpotent. + */ private static A sparseFold(String cacheName, IgniteBiFunction, A, A> folder, IgnitePredicate keyFilter, BinaryOperator accumulator, A zeroVal, V defVal, K defKey, long defValCnt, boolean isNilpotent) { @@ -411,7 +519,7 @@ private static A sparseFold(String cacheName, IgniteBiFunction A sparseFold(String cacheName, IgniteBiFunction entry : cache.query(new ScanQuery(part, - (k, v) -> affinity.mapPartitionToNode(p) == localNode && (keyFilter == null || keyFilter.apply(k))))) + (k, v) -> affinity.mapPartitionToNode(p) == locNode && (keyFilter == null || keyFilter.apply(k))))) a = folder.apply(entry, a); } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java index d1d3904f0be2f..3dc9b43a0ad89 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java @@ -503,7 +503,7 @@ private void checkCardinality(int rows, int cols) { /** {@inheritDoc} */ @Override public double determinant() { - //TODO: This decomposition should be cached + //TODO: IGNITE-5799, This decomposition should be cached LUDecomposition dec = new LUDecomposition(this); double res = dec.determinant(); dec.destroy(); @@ -515,7 +515,7 @@ private void checkCardinality(int rows, int cols) { if (rowSize() != columnSize()) throw new CardinalityException(rowSize(), columnSize()); - //TODO: This decomposition should be cached + //TODO: IGNITE-5799, This decomposition should be cached LUDecomposition dec = new LUDecomposition(this); Matrix res = dec.solve(likeIdentity()); diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/BlockEntry.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/BlockEntry.java new file mode 100644 index 0000000000000..47f07ce6a99b1 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/BlockEntry.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.impls.matrix; + +import org.apache.ignite.ml.math.Matrix; + +/** + * Block for {@link SparseBlockDistributedMatrix}. + */ +public final class BlockEntry extends SparseLocalOnHeapMatrix { + /** Max block size. */ + public static final int MAX_BLOCK_SIZE = 32; + + /** */ + public BlockEntry() { + // No-op. + } + + /** */ + public BlockEntry(int row, int col) { + super(row, col); + + assert col <= MAX_BLOCK_SIZE; + assert row <= MAX_BLOCK_SIZE; + } + + /** */ + public BlockEntry(Matrix mtx) { + assert mtx.columnSize() <= MAX_BLOCK_SIZE; + assert mtx.rowSize() <= MAX_BLOCK_SIZE; + + setStorage(mtx.getStorage()); + } + +} diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrix.java index a7f0afcfc3afa..7f00bcb276328 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrix.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrix.java @@ -64,7 +64,6 @@ public CacheMatrix( } /** - * * */ @SuppressWarnings({"unchecked"}) @@ -93,7 +92,7 @@ private CacheMatrixStorage storage() { * @param d Value to divide to. */ @Override public Matrix divide(double d) { - return mapOverValues((Double v) -> v / d); + return mapOverValues(v -> v / d); } /** @@ -102,7 +101,7 @@ private CacheMatrixStorage storage() { * @param x Value to add. */ @Override public Matrix plus(double x) { - return mapOverValues((Double v) -> v + x); + return mapOverValues(v -> v + x); } /** @@ -111,12 +110,12 @@ private CacheMatrixStorage storage() { * @param x Value to multiply to. */ @Override public Matrix times(double x) { - return mapOverValues((Double v) -> v * x); + return mapOverValues(v -> v * x); } /** {@inheritDoc} */ @Override public Matrix assign(double val) { - return mapOverValues((Double v) -> val); + return mapOverValues(v -> val); } /** {@inheritDoc} */ diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseBlockDistributedMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseBlockDistributedMatrix.java new file mode 100644 index 0000000000000..b3481f92ebf30 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseBlockDistributedMatrix.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.impls.matrix; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.Ignition; +import org.apache.ignite.cache.affinity.Affinity; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.lang.IgniteUuid; +import org.apache.ignite.ml.math.Matrix; +import org.apache.ignite.ml.math.StorageConstants; +import org.apache.ignite.ml.math.Vector; +import org.apache.ignite.ml.math.exceptions.CardinalityException; +import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException; +import org.apache.ignite.ml.math.functions.IgniteDoubleFunction; +import org.apache.ignite.ml.math.impls.CacheUtils; +import org.apache.ignite.ml.math.impls.storage.matrix.BlockMatrixKey; +import org.apache.ignite.ml.math.impls.storage.matrix.BlockMatrixStorage; + +/** + * Sparse block distributed matrix. This matrix represented by blocks 32x32 {@link BlockEntry}. + * + * Using separate cache with keys {@link BlockMatrixKey} and values {@link BlockEntry}. + */ +public class SparseBlockDistributedMatrix extends AbstractMatrix implements StorageConstants { + /** + * + */ + public SparseBlockDistributedMatrix() { + // No-op. + } + + /** + * @param rows Amount of rows in the matrix. + * @param cols Amount of columns in the matrix. + */ + public SparseBlockDistributedMatrix(int rows, int cols) { + assert rows > 0; + assert cols > 0; + + setStorage(new BlockMatrixStorage(rows, cols)); + } + + /** + * Return the same matrix with updates values (broken contract). + * + * @param d Value to divide to. + */ + @Override public Matrix divide(double d) { + return mapOverValues(v -> v / d); + } + + /** + * Return the same matrix with updates values (broken contract). + * + * @param x Value to add. + */ + @Override public Matrix plus(double x) { + return mapOverValues(v -> v + x); + } + + /** + * Return the same matrix with updates values (broken contract). + * + * @param x Value to multiply. + */ + @Override public Matrix times(double x) { + return mapOverValues(v -> v * x); + } + + /** + * {@inheritDoc} + */ + @SuppressWarnings({"unchecked"}) + @Override public Matrix times(final Matrix mtx) { + if (mtx == null) + throw new IllegalArgumentException("The matrix should be not null."); + + if (columnSize() != mtx.rowSize()) + throw new CardinalityException(columnSize(), mtx.rowSize()); + + SparseBlockDistributedMatrix matrixA = this; + SparseBlockDistributedMatrix matrixB = (SparseBlockDistributedMatrix)mtx; + + String cacheName = BlockMatrixStorage.ML_BLOCK_CACHE_NAME; + SparseBlockDistributedMatrix matrixC = new SparseBlockDistributedMatrix(matrixA.rowSize(), matrixB.columnSize()); + + CacheUtils.bcast(BlockMatrixStorage.ML_BLOCK_CACHE_NAME, () -> { + Ignite ignite = Ignition.localIgnite(); + Affinity affinity = ignite.affinity(cacheName); + + IgniteCache cache = ignite.getOrCreateCache(cacheName); + ClusterNode locNode = ignite.cluster().localNode(); + + BlockMatrixStorage storageC = matrixC.storage(); + + Map> keysCToNodes = affinity.mapKeysToNodes(storageC.getAllKeys()); + Collection locKeys = keysCToNodes.get(locNode); + + if (locKeys == null) + return; + + // compute Cij locally on each node + // TODO: IGNITE:5114, exec in parallel + locKeys.forEach(key -> { + long newBlockId = key.blockId(); + BlockEntry blockC = null; + + List aRow = matrixA.storage().getRowForBlock(newBlockId, storageC); + List bCol = matrixB.storage().getColForBlock(newBlockId, storageC); + + for (int i = 0; i < aRow.size(); i++) { + BlockEntry blockA = aRow.get(i); + BlockEntry blockB = bCol.get(i); + + BlockEntry tmpBlock = new BlockEntry(blockA.times(blockB)); + + blockC = blockC == null ? tmpBlock : new BlockEntry(blockC.plus(tmpBlock)); + } + + cache.put(storageC.getCacheKey(newBlockId), blockC); + }); + }); + + return matrixC; + } + + /** {@inheritDoc} */ + @Override public Matrix assign(double val) { + return mapOverValues(v -> val); + } + + /** {@inheritDoc} */ + @Override public Matrix map(IgniteDoubleFunction fun) { + return mapOverValues(fun); + } + + /** {@inheritDoc} */ + @Override public double sum() { + return CacheUtils.sparseSum(getUUID(), BlockMatrixStorage.ML_BLOCK_CACHE_NAME); + } + + /** {@inheritDoc} */ + @Override public double maxValue() { + return CacheUtils.sparseMax(getUUID(), BlockMatrixStorage.ML_BLOCK_CACHE_NAME); + } + + /** {@inheritDoc} */ + @Override public double minValue() { + return CacheUtils.sparseMin(getUUID(), BlockMatrixStorage.ML_BLOCK_CACHE_NAME); + } + + /** {@inheritDoc} */ + @Override public Matrix copy() { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override public Matrix like(int rows, int cols) { + return new SparseBlockDistributedMatrix(rows, cols); + } + + /** {@inheritDoc} */ + @Override public Vector likeVector(int crd) { + throw new UnsupportedOperationException(); + } + + /** */ + private IgniteUuid getUUID() { + return ((BlockMatrixStorage)getStorage()).getUUID(); + } + + /** + * @param mapper Mapping function. + * @return Matrix with mapped values. + */ + private Matrix mapOverValues(IgniteDoubleFunction mapper) { + CacheUtils.sparseMap(getUUID(), mapper, BlockMatrixStorage.ML_BLOCK_CACHE_NAME); + + return this; + } + + /** + * + */ + private BlockMatrixStorage storage() { + return (BlockMatrixStorage)getStorage(); + } +} diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java index df2ddc4b1c28c..a86db95f18d84 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java @@ -23,7 +23,6 @@ import org.apache.ignite.ml.math.Vector; import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException; import org.apache.ignite.ml.math.functions.IgniteDoubleFunction; -import org.apache.ignite.ml.math.functions.IgniteFunction; import org.apache.ignite.ml.math.impls.CacheUtils; import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage; @@ -61,10 +60,7 @@ public SparseDistributedMatrix(int rows, int cols, int stoMode, int acsMode) { setStorage(new SparseDistributedMatrixStorage(rows, cols, stoMode, acsMode)); } - /** - * - * - */ + /** */ private SparseDistributedMatrixStorage storage() { return (SparseDistributedMatrixStorage)getStorage(); } @@ -75,7 +71,7 @@ private SparseDistributedMatrixStorage storage() { * @param d Value to divide to. */ @Override public Matrix divide(double d) { - return mapOverValues((Double v) -> v / d); + return mapOverValues(v -> v / d); } /** @@ -84,7 +80,7 @@ private SparseDistributedMatrixStorage storage() { * @param x Value to add. */ @Override public Matrix plus(double x) { - return mapOverValues((Double v) -> v + x); + return mapOverValues(v -> v + x); } /** @@ -93,42 +89,42 @@ private SparseDistributedMatrixStorage storage() { * @param x Value to multiply. */ @Override public Matrix times(double x) { - return mapOverValues((Double v) -> v * x); + return mapOverValues(v -> v * x); } /** {@inheritDoc} */ @Override public Matrix assign(double val) { - return mapOverValues((Double v) -> val); + return mapOverValues(v -> val); } /** {@inheritDoc} */ @Override public Matrix map(IgniteDoubleFunction fun) { - return mapOverValues(fun::apply); + return mapOverValues(fun); } /** * @param mapper Mapping function. * @return Matrix with mapped values. */ - private Matrix mapOverValues(IgniteFunction mapper) { - CacheUtils.sparseMap(getUUID(), mapper); + private Matrix mapOverValues(IgniteDoubleFunction mapper) { + CacheUtils.sparseMap(getUUID(), mapper, SparseDistributedMatrixStorage.ML_CACHE_NAME); return this; } /** {@inheritDoc} */ @Override public double sum() { - return CacheUtils.sparseSum(getUUID()); + return CacheUtils.sparseSum(getUUID(), SparseDistributedMatrixStorage.ML_CACHE_NAME); } /** {@inheritDoc} */ @Override public double maxValue() { - return CacheUtils.sparseMax(getUUID()); + return CacheUtils.sparseMax(getUUID(), SparseDistributedMatrixStorage.ML_CACHE_NAME); } /** {@inheritDoc} */ @Override public double minValue() { - return CacheUtils.sparseMin(getUUID()); + return CacheUtils.sparseMin(getUUID(), SparseDistributedMatrixStorage.ML_CACHE_NAME); } /** {@inheritDoc} */ diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BaseBlockMatrixKey.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BaseBlockMatrixKey.java new file mode 100644 index 0000000000000..74ddfe58c62f6 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BaseBlockMatrixKey.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.impls.storage.matrix; + +import org.apache.ignite.lang.IgniteUuid; +import org.apache.ignite.ml.math.impls.matrix.SparseBlockDistributedMatrix; + +/** + * Cache key for blocks in {@link SparseBlockDistributedMatrix}. + */ +public interface BaseBlockMatrixKey { + /** + * @return block id. + */ + public long blockId(); + + /** + * @return matrix id. + */ + public IgniteUuid matrixId(); + + /** + * @return key affinity key. + */ + public IgniteUuid affinityKey(); +} diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixKey.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixKey.java new file mode 100644 index 0000000000000..3749f44a2f7f4 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixKey.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.impls.storage.matrix; + +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import org.apache.ignite.binary.BinaryObjectException; +import org.apache.ignite.binary.BinaryRawReader; +import org.apache.ignite.binary.BinaryRawWriter; +import org.apache.ignite.binary.BinaryReader; +import org.apache.ignite.binary.BinaryWriter; +import org.apache.ignite.binary.Binarylizable; +import org.apache.ignite.internal.binary.BinaryUtils; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteUuid; +import org.apache.ignite.ml.math.impls.matrix.BlockEntry; +import org.apache.ignite.ml.math.impls.matrix.SparseBlockDistributedMatrix; +import org.jetbrains.annotations.Nullable; + +/** + * Key implementation for {@link BlockEntry} using for {@link SparseBlockDistributedMatrix}. + */ +public class BlockMatrixKey implements BaseBlockMatrixKey, Externalizable, Binarylizable { + /** */ + private static final long serialVersionUID = 0L; + /** Block ID */ + private long blockId; + /** Matrix ID */ + private IgniteUuid matrixUuid; + /** Block affinity key. */ + private IgniteUuid affinityKey; + + /** + * Empty constructor required for {@link Externalizable}. + */ + public BlockMatrixKey() { + // No-op. + } + + /** + * Construct matrix block key. + * + * @param blockId Block id. + * @param matrixUuid Matrix uuid. + * @param affinityKey Affinity key. + */ + public BlockMatrixKey(long blockId, IgniteUuid matrixUuid, @Nullable IgniteUuid affinityKey) { + assert blockId >= 0; + assert matrixUuid != null; + + this.blockId = blockId; + this.matrixUuid = matrixUuid; + this.affinityKey = affinityKey; + } + + /** {@inheritDoc} */ + @Override public long blockId() { + return blockId; + } + + /** {@inheritDoc} */ + @Override public IgniteUuid matrixId() { + return matrixUuid; + } + + /** {@inheritDoc} */ + @Override public IgniteUuid affinityKey() { + return affinityKey; + } + + /** {@inheritDoc} */ + @Override public void writeExternal(ObjectOutput out) throws IOException { + U.writeGridUuid(out, matrixUuid); + U.writeGridUuid(out, affinityKey); + out.writeLong(blockId); + } + + /** {@inheritDoc} */ + @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + matrixUuid = U.readGridUuid(in); + affinityKey = U.readGridUuid(in); + blockId = in.readLong(); + } + + /** {@inheritDoc} */ + @Override public void writeBinary(BinaryWriter writer) throws BinaryObjectException { + BinaryRawWriter out = writer.rawWriter(); + + BinaryUtils.writeIgniteUuid(out, matrixUuid); + BinaryUtils.writeIgniteUuid(out, affinityKey); + out.writeLong(blockId); + } + + /** {@inheritDoc} */ + @Override public void readBinary(BinaryReader reader) throws BinaryObjectException { + BinaryRawReader in = reader.rawReader(); + + matrixUuid = BinaryUtils.readIgniteUuid(in); + affinityKey = BinaryUtils.readIgniteUuid(in); + blockId = in.readLong(); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return matrixUuid.hashCode() + (int)(blockId ^ (blockId >>> 32)); + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object obj) { + if (obj == this) + return true; + + if (obj == null || obj.getClass() != getClass()) + return false; + + BlockMatrixKey that = (BlockMatrixKey)obj; + + return blockId == that.blockId && matrixUuid.equals(that.matrixUuid) && F.eq(affinityKey, that.affinityKey); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(BlockMatrixKey.class, this); + } +} diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java new file mode 100644 index 0000000000000..6640e5acdab6d --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java @@ -0,0 +1,435 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.impls.storage.matrix; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.LongStream; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.Ignition; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.CachePeekMode; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteUuid; +import org.apache.ignite.ml.math.MatrixStorage; +import org.apache.ignite.ml.math.StorageConstants; +import org.apache.ignite.ml.math.impls.CacheUtils; +import org.apache.ignite.ml.math.impls.matrix.BlockEntry; +import org.apache.ignite.ml.math.impls.matrix.SparseBlockDistributedMatrix; + +import static org.apache.ignite.ml.math.impls.matrix.BlockEntry.MAX_BLOCK_SIZE; + +/** + * Storage for {@link SparseBlockDistributedMatrix}. + */ +public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, StorageConstants { + /** Cache name used for all instances of {@link BlockMatrixStorage}. */ + public static final String ML_BLOCK_CACHE_NAME = "ML_BLOCK_SPARSE_MATRICES_CONTAINER"; + /** */ + private int blocksInCol; + /** */ + private int blocksInRow; + /** Amount of rows in the matrix. */ + private int rows; + /** Amount of columns in the matrix. */ + private int cols; + /** Matrix uuid. */ + private IgniteUuid uuid; + /** Block size about 8 KB of data. */ + private int maxBlockEdge = MAX_BLOCK_SIZE; + + /** Actual distributed storage. */ + private IgniteCache< + BlockMatrixKey /* Matrix block number with uuid. */, + BlockEntry /* Block of matrix, local sparse matrix. */ + > cache = null; + + /** + * + */ + public BlockMatrixStorage() { + // No-op. + } + + /** + * @param rows Amount of rows in the matrix. + * @param cols Amount of columns in the matrix. + */ + public BlockMatrixStorage(int rows, int cols) { + assert rows > 0; + assert cols > 0; + + this.rows = rows; + this.cols = cols; + + //cols % maxBlockEdge > 0 ? 1 : 0 + + this.blocksInRow = cols % maxBlockEdge == 0 ? cols / maxBlockEdge : cols / maxBlockEdge + 1; + this.blocksInCol = rows % maxBlockEdge == 0 ? rows / maxBlockEdge : rows / maxBlockEdge + 1; + + cache = newCache(); + + uuid = IgniteUuid.randomUuid(); + } + + /** + * + */ + public IgniteCache cache() { + return cache; + } + + /** {@inheritDoc} */ + @Override public double get(int x, int y) { + return matrixGet(x, y); + } + + /** {@inheritDoc} */ + @Override public void set(int x, int y, double v) { + matrixSet(x, y, v); + } + + /** {@inheritDoc} */ + @Override public int columnSize() { + return cols; + } + + /** {@inheritDoc} */ + @Override public int rowSize() { + return rows; + } + + /** {@inheritDoc} */ + @Override public void writeExternal(ObjectOutput out) throws IOException { + out.writeInt(rows); + out.writeInt(cols); + out.writeInt(blocksInRow); + out.writeInt(blocksInCol); + U.writeGridUuid(out, uuid); + out.writeUTF(cache.getName()); + } + + /** {@inheritDoc} */ + @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + rows = in.readInt(); + cols = in.readInt(); + blocksInRow = in.readInt(); + blocksInCol = in.readInt(); + uuid = U.readGridUuid(in); + cache = ignite().getOrCreateCache(in.readUTF()); + } + + /** {@inheritDoc} */ + @Override public boolean isSequentialAccess() { + return false; + } + + /** {@inheritDoc} */ + @Override public boolean isDense() { + return false; + } + + /** {@inheritDoc} */ + @Override public boolean isRandomAccess() { + return true; + } + + /** {@inheritDoc} */ + @Override public boolean isDistributed() { + return true; + } + + /** {@inheritDoc} */ + @Override public boolean isArrayBased() { + return false; + } + + /** Delete all data from cache. */ + @Override public void destroy() { + long maxBlockId = getBlockId(cols, rows); + + Set keyset = LongStream.rangeClosed(0, maxBlockId).mapToObj(this::getCacheKey).collect(Collectors.toSet()); + + cache.clearAll(keyset); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + int res = 1; + + res = res * 37 + cols; + res = res * 37 + rows; + res = res * 37 + uuid.hashCode(); + res = res * 37 + cache.hashCode(); + + return res; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object obj) { + if (this == obj) + return true; + + if (obj == null || getClass() != obj.getClass()) + return false; + + BlockMatrixStorage that = (BlockMatrixStorage)obj; + + return rows == that.rows && cols == that.cols && uuid.equals(that.uuid) + && (cache != null ? cache.equals(that.cache) : that.cache == null); + } + + /** + * Get storage UUID. + * + * @return storage UUID. + */ + public IgniteUuid getUUID() { + return uuid; + } + + /** + * Build the cache key for the given block id + */ + public BlockMatrixKey getCacheKey(long blockId) { + return new BlockMatrixKey(blockId, uuid, getAffinityKey(blockId)); + } + + /** + * Get rows for current block. + * + * @param blockId block id. + * @param storageC result storage. + * @return The list of block entries. + */ + public List getRowForBlock(long blockId, BlockMatrixStorage storageC) { + long blockRow = blockId / storageC.blocksInCol; + long blockCol = blockId % storageC.blocksInRow; + + long locBlock = this.blocksInRow * (blockRow) + (blockCol >= this.blocksInRow ? (blocksInRow - 1) : blockCol); + + return getRowForBlock(locBlock); + } + + /** + * Get cols for current block. + * + * @param blockId block id. + * @param storageC result storage. + * @return The list of block entries. + */ + public List getColForBlock(long blockId, BlockMatrixStorage storageC) { + long blockRow = blockId / storageC.blocksInCol; + long blockCol = blockId % storageC.blocksInRow; + + long locBlock = this.blocksInRow * (blockRow) + (blockCol >= this.blocksInRow ? (blocksInRow - 1) : blockCol); + + return getColForBlock(locBlock); + } + + /** + * Build a keyset for this matrix storage. + */ + public Collection getAllKeys() { + long maxBlockId = numberOfBlocks(); + Collection keys = new LinkedList<>(); + + for (long id = 0; id < maxBlockId; id++) + keys.add(getCacheKey(id)); + + return keys; + } + + /** */ + private List getRowForBlock(long blockId) { + List res = new LinkedList<>(); + + boolean isFirstRow = blockId < blocksInRow; + + long startBlock = isFirstRow ? 0 : blockId - blockId % blocksInRow; + long endBlock = startBlock + blocksInRow - 1; + + for (long i = startBlock; i <= endBlock; i++) + res.add(getEntryById(i)); + + return res; + } + + /** */ + private List getColForBlock(long blockId) { + List res = new LinkedList<>(); + + long startBlock = blockId % blocksInRow; + long endBlock = startBlock + blocksInRow * (blocksInCol - 1); + + for (long i = startBlock; i <= endBlock; i += blocksInRow) + res.add(getEntryById(i)); + + return res; + } + + /** + * + */ + private BlockEntry getEntryById(long blockId) { + BlockMatrixKey key = getCacheKey(blockId); + + BlockEntry entry = cache.localPeek(key); + entry = entry != null ? entry : cache.get(key); + + if (entry == null) { + long colId = blockId == 0 ? 0 : blockId + 1; + + boolean isLastRow = (blockId) >= blocksInRow * (blocksInCol - 1); + boolean isLastCol = (colId) % blocksInRow == 0; + + entry = new BlockEntry(isLastRow && rows % maxBlockEdge != 0 ? rows % maxBlockEdge : maxBlockEdge, isLastCol && cols % maxBlockEdge != 0 ? cols % maxBlockEdge : maxBlockEdge); + } + + return entry; + } + + /** + * + */ + private long numberOfBlocks() { + int rows = rowSize(); + int cols = columnSize(); + + return ((rows / maxBlockEdge) + (((rows % maxBlockEdge) > 0) ? 1 : 0)) + * ((cols / maxBlockEdge) + (((cols % maxBlockEdge) > 0) ? 1 : 0)); + } + + /** + * TODO: IGNITE-5646, WIP + * + * Get affinity key for the given id. + */ + private IgniteUuid getAffinityKey(long id) { + return null; + } + + /** + * Distributed matrix set. + * + * @param a Row or column index. + * @param b Row or column index. + * @param v New value to set. + */ + private void matrixSet(int a, int b, double v) { + long id = getBlockId(a, b); + // Remote set on the primary node (where given row or column is stored locally). + ignite().compute(groupForKey(ML_BLOCK_CACHE_NAME, id)).run(() -> { + IgniteCache cache = Ignition.localIgnite().getOrCreateCache(ML_BLOCK_CACHE_NAME); + + BlockMatrixKey key = getCacheKey(getBlockId(a, b)); + + // Local get. + BlockEntry block = cache.localPeek(key, CachePeekMode.PRIMARY); + + if (block == null) + block = cache.get(key); //Remote entry get. + + if (block == null) + block = initBlockFor(a, b); + + block.set(a % block.rowSize(), b % block.columnSize(), v); + + // Local put. + cache.put(key, block); + }); + } + + /** */ + private long getBlockId(int x, int y) { + return (y / maxBlockEdge) * blockShift(cols) + (x / maxBlockEdge); + } + + /** */ + private BlockEntry initBlockFor(int x, int y) { + int blockRows = rows - x >= maxBlockEdge ? maxBlockEdge : rows - x; + int blockCols = cols - y >= maxBlockEdge ? maxBlockEdge : cols - y; + + return new BlockEntry(blockRows, blockCols); + } + + /** */ + private int blockShift(int i) { + return (i) / maxBlockEdge + ((i) % maxBlockEdge > 0 ? 1 : 0); + } + + /** + * Distributed matrix get. + * + * @param a Row or column index. + * @param b Row or column index. + * @return Matrix value at (a, b) index. + */ + private double matrixGet(int a, int b) { + // Remote get from the primary node (where given row or column is stored locally). + return ignite().compute(groupForKey(ML_BLOCK_CACHE_NAME, getBlockId(a, b))).call(() -> { + IgniteCache cache = Ignition.localIgnite().getOrCreateCache(ML_BLOCK_CACHE_NAME); + + BlockMatrixKey key = getCacheKey(getBlockId(a, b)); + + // Local get. + BlockEntry block = cache.localPeek(key, CachePeekMode.PRIMARY); + + if (block == null) + block = cache.get(key); + + return block == null ? 0.0 : block.get(a % block.rowSize(), b % block.columnSize()); + }); + } + + /** + * Create new ML cache if needed. + */ + private IgniteCache newCache() { + CacheConfiguration cfg = new CacheConfiguration<>(); + + // Write to primary. + cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC); + + // Atomic transactions only. + cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC); + + // No eviction. + cfg.setEvictionPolicy(null); + + // No copying of values. + cfg.setCopyOnRead(false); + + // Cache is partitioned. + cfg.setCacheMode(CacheMode.PARTITIONED); + + // Random cache name. + cfg.setName(ML_BLOCK_CACHE_NAME); + + return Ignition.localIgnite().getOrCreateCache(cfg); + } +} diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java index f2efe74121348..5145376630066 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java @@ -46,9 +46,7 @@ public SparseLocalOnHeapVectorStorage() { // No-op. } - /** - * @param map - */ + /** */ public SparseLocalOnHeapVectorStorage(Map map, boolean copy) { assert map.size() > 0; diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/Variance.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/Variance.java index e406b5b81a95e..525e6e924f2fd 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/Variance.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/Variance.java @@ -30,6 +30,7 @@ public class Variance { /** */ private double m2; + /** */ public Variance() { mean = 0; n = 0; diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/package-info.java new file mode 100644 index 0000000000000..7b65fce86133e --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + * Statistics stuff. + */ +package org.apache.ignite.ml.math.statistics; \ No newline at end of file diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MapUtil.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MapUtil.java index 6c25f0e53c679..9190901bf215e 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MapUtil.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MapUtil.java @@ -25,7 +25,7 @@ import java.util.stream.Stream; /** - * + * Some {@link Map} related utils. */ public class MapUtil { /** */ diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/package-info.java new file mode 100644 index 0000000000000..2507ee4f7ff4f --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + * Some math utils. + */ +package org.apache.ignite.ml.math.util; \ No newline at end of file diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/package-info.java new file mode 100644 index 0000000000000..779581b2473a1 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + * Root ML package. + */ +package org.apache.ignite.ml; \ No newline at end of file diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplDistributedTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplDistributedTestSuite.java index 9899d3b79c06a..5dc860c7128a9 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplDistributedTestSuite.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplDistributedTestSuite.java @@ -18,6 +18,7 @@ package org.apache.ignite.ml.math; import org.apache.ignite.ml.math.impls.matrix.CacheMatrixTest; +import org.apache.ignite.ml.math.impls.matrix.SparseDistributedBlockMatrixTest; import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrixTest; import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorageTest; import org.apache.ignite.ml.math.impls.vector.CacheVectorTest; @@ -33,6 +34,7 @@ CacheMatrixTest.class, SparseDistributedMatrixStorageTest.class, SparseDistributedMatrixTest.class, + SparseDistributedBlockMatrixTest.class }) public class MathImplDistributedTestSuite { // No-op. diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java new file mode 100644 index 0000000000000..1228f059630f0 --- /dev/null +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java @@ -0,0 +1,379 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.impls.matrix; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.internal.util.IgniteUtils; +import org.apache.ignite.lang.IgniteUuid; +import org.apache.ignite.ml.math.Matrix; +import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException; +import org.apache.ignite.ml.math.impls.MathTestConstants; +import org.apache.ignite.ml.math.impls.storage.matrix.BlockMatrixKey; +import org.apache.ignite.ml.math.impls.storage.matrix.BlockMatrixStorage; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.testframework.junits.common.GridCommonTest; + +import static org.apache.ignite.ml.math.impls.MathTestConstants.UNEXPECTED_VAL; + +/** + * Tests for {@link SparseBlockDistributedMatrix}. + */ +@GridCommonTest(group = "Distributed Models") +public class SparseDistributedBlockMatrixTest extends GridCommonAbstractTest { + /** Number of nodes in grid */ + private static final int NODE_COUNT = 3; + /** Precision. */ + private static final double PRECISION = 0.0; + /** Grid instance. */ + private Ignite ignite; + /** Matrix rows */ + private final int rows = MathTestConstants.STORAGE_SIZE; + /** Matrix cols */ + private final int cols = MathTestConstants.STORAGE_SIZE; + /** Matrix for tests */ + private SparseBlockDistributedMatrix cacheMatrix; + + /** + * Default constructor. + */ + public SparseDistributedBlockMatrixTest() { + super(false); + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + for (int i = 1; i <= NODE_COUNT; i++) + startGrid(i); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + } + + /** + * {@inheritDoc} + */ + @Override protected void beforeTest() throws Exception { + ignite = grid(NODE_COUNT); + + ignite.configuration().setPeerClassLoadingEnabled(true); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + if (cacheMatrix != null) { + cacheMatrix.destroy(); + cacheMatrix = null; + } + } + + /** */ + public void testGetSet() throws Exception { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + cacheMatrix = new SparseBlockDistributedMatrix(rows, cols); + + for (int i = 0; i < rows; i++) { + for (int j = 0; j < cols; j++) { + double v = Math.random(); + cacheMatrix.set(i, j, v); + + assertEquals("Unexpected value for matrix element["+ i +" " + j + "]", v, cacheMatrix.get(i, j), PRECISION); + } + } + } + + /** */ + public void testExternalize() throws IOException, ClassNotFoundException { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + cacheMatrix = new SparseBlockDistributedMatrix(rows, cols); + + cacheMatrix.set(1, 1, 1.0); + + ByteArrayOutputStream byteArrOutputStream = new ByteArrayOutputStream(); + ObjectOutputStream objOutputStream = new ObjectOutputStream(byteArrOutputStream); + + objOutputStream.writeObject(cacheMatrix); + + ByteArrayInputStream byteArrInputStream = new ByteArrayInputStream(byteArrOutputStream.toByteArray()); + ObjectInputStream objInputStream = new ObjectInputStream(byteArrInputStream); + + SparseBlockDistributedMatrix objRestored = (SparseBlockDistributedMatrix)objInputStream.readObject(); + + assertTrue(MathTestConstants.VAL_NOT_EQUALS, cacheMatrix.equals(objRestored)); + assertEquals(MathTestConstants.VAL_NOT_EQUALS, objRestored.get(1, 1), 1.0, PRECISION); + } + + /** Test simple math. */ + public void testMath() { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + cacheMatrix = new SparseBlockDistributedMatrix(rows, cols); + initMtx(cacheMatrix); + + cacheMatrix.assign(2.0); + for (int i = 0; i < cacheMatrix.rowSize(); i++) + for (int j = 0; j < cacheMatrix.columnSize(); j++) + assertEquals(UNEXPECTED_VAL, 2.0, cacheMatrix.get(i, j), PRECISION); + + cacheMatrix.plus(3.0); + for (int i = 0; i < cacheMatrix.rowSize(); i++) + for (int j = 0; j < cacheMatrix.columnSize(); j++) + assertEquals(UNEXPECTED_VAL, 5.0, cacheMatrix.get(i, j), PRECISION); + + cacheMatrix.times(2.0); + for (int i = 0; i < cacheMatrix.rowSize(); i++) + for (int j = 0; j < cacheMatrix.columnSize(); j++) + assertEquals(UNEXPECTED_VAL, 10.0, cacheMatrix.get(i, j), PRECISION); + + cacheMatrix.divide(10.0); + for (int i = 0; i < cacheMatrix.rowSize(); i++) + for (int j = 0; j < cacheMatrix.columnSize(); j++) + assertEquals(UNEXPECTED_VAL, 1.0, cacheMatrix.get(i, j), PRECISION); + + assertEquals(UNEXPECTED_VAL, cacheMatrix.rowSize() * cacheMatrix.columnSize(), cacheMatrix.sum(), PRECISION); + } + + /** */ + public void testMinMax() { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + cacheMatrix = new SparseBlockDistributedMatrix(rows, cols); + + for (int i = 0; i < cacheMatrix.rowSize(); i++) + for (int j = 0; j < cacheMatrix.columnSize(); j++) + cacheMatrix.set(i, j, i * cols + j + 1); + + assertEquals(UNEXPECTED_VAL, 1.0, cacheMatrix.minValue(), PRECISION); + assertEquals(UNEXPECTED_VAL, rows * cols, cacheMatrix.maxValue(), PRECISION); + + for (int i = 0; i < cacheMatrix.rowSize(); i++) + for (int j = 0; j < cacheMatrix.columnSize(); j++) + cacheMatrix.set(i, j, -1.0 * (i * cols + j + 1)); + + assertEquals(UNEXPECTED_VAL, -rows * cols, cacheMatrix.minValue(), PRECISION); + assertEquals(UNEXPECTED_VAL, -1.0, cacheMatrix.maxValue(), PRECISION); + + for (int i = 0; i < cacheMatrix.rowSize(); i++) + for (int j = 0; j < cacheMatrix.columnSize(); j++) + cacheMatrix.set(i, j, i * cols + j); + + assertEquals(UNEXPECTED_VAL, 0.0, cacheMatrix.minValue(), PRECISION); + assertEquals(UNEXPECTED_VAL, rows * cols - 1.0, cacheMatrix.maxValue(), PRECISION); + } + + /** */ + public void testMap() { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + cacheMatrix = new SparseBlockDistributedMatrix(rows, cols); + initMtx(cacheMatrix); + + cacheMatrix.map(i -> 100.0); + for (int i = 0; i < cacheMatrix.rowSize(); i++) + for (int j = 0; j < cacheMatrix.columnSize(); j++) + assertEquals(UNEXPECTED_VAL, 100.0, cacheMatrix.get(i, j), PRECISION); + } + + /** */ + public void testCopy() { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + cacheMatrix = new SparseBlockDistributedMatrix(rows, cols); + + try { + cacheMatrix.copy(); + fail("UnsupportedOperationException expected."); + } + catch (UnsupportedOperationException e) { + return; + } + fail("UnsupportedOperationException expected."); + } + + /** */ + public void testCacheBehaviour(){ + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + SparseBlockDistributedMatrix cacheMatrix1 = new SparseBlockDistributedMatrix(rows, cols); + SparseBlockDistributedMatrix cacheMatrix2 = new SparseBlockDistributedMatrix(rows, cols); + + initMtx(cacheMatrix1); + initMtx(cacheMatrix2); + + Collection cacheNames = ignite.cacheNames(); + + assert cacheNames.contains(BlockMatrixStorage.ML_BLOCK_CACHE_NAME); + + IgniteCache cache = ignite.getOrCreateCache(BlockMatrixStorage.ML_BLOCK_CACHE_NAME); + + Set keySet1 = buildKeySet(cacheMatrix1); + Set keySet2 = buildKeySet(cacheMatrix2); + + assert cache.containsKeys(keySet1); + assert cache.containsKeys(keySet2); + + cacheMatrix2.destroy(); + + assert cache.containsKeys(keySet1); + assert !cache.containsKeys(keySet2); + + cacheMatrix1.destroy(); + + assert !cache.containsKeys(keySet1); + } + + /** */ + public void testLike() { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + cacheMatrix = new SparseBlockDistributedMatrix(rows, cols); + + assertNotNull(cacheMatrix.like(1, 1)); + } + + /** */ + public void testLikeVector() { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + cacheMatrix = new SparseBlockDistributedMatrix(rows, cols); + + try { + cacheMatrix.likeVector(1); + fail("UnsupportedOperationException expected."); + } + catch (UnsupportedOperationException e) { + return; + } + fail("UnsupportedOperationException expected."); + } + + /** + * Simple test for two square matrices. + */ + public void testSquareMatrixTimes(){ + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + int size = 100; + + Matrix cacheMatrix1 = new SparseBlockDistributedMatrix(size, size); + Matrix cacheMatrix2 = new SparseBlockDistributedMatrix(size, size); + + for (int i = 0; i < size; i++) { + cacheMatrix1.setX(i, i, i); + cacheMatrix2.setX(i, i, i); + } + + Matrix res = cacheMatrix1.times(cacheMatrix2); + + for(int i = 0; i < size; i++) + for(int j = 0; j < size; j++) + if (i == j) + assertEquals(UNEXPECTED_VAL + " for "+ i +":"+ j, i * i, res.get(i, j), PRECISION); + else + assertEquals(UNEXPECTED_VAL + " for "+ i +":"+ j, 0, res.get(i, j), PRECISION); + } + + /** + * + */ + public void testNonSquareMatrixTimes(){ + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + int size = BlockEntry.MAX_BLOCK_SIZE + 1; + int size2 = BlockEntry.MAX_BLOCK_SIZE * 2 + 1; + + Matrix cacheMatrix1 = new SparseBlockDistributedMatrix(size2, size); + Matrix cacheMatrix2 = new SparseBlockDistributedMatrix(size, size2); + + for (int i = 0; i < size; i++) { + cacheMatrix1.setX(i, i, i); + cacheMatrix2.setX(i, i, i); + } + + Matrix res = cacheMatrix1.times(cacheMatrix2); + + for(int i = 0; i < size; i++) + for(int j = 0; j < size; j++) + if (i == j) + assertEquals(UNEXPECTED_VAL + " for "+ i +":"+ j, i * i, res.get(i, j), PRECISION); + else + assertEquals(UNEXPECTED_VAL + " for "+ i +":"+ j, 0, res.get(i, j), PRECISION); + } + + /** + * + */ + public void testNonSquareMatrixTimes2(){ + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + int size = BlockEntry.MAX_BLOCK_SIZE + 1; + int size2 = BlockEntry.MAX_BLOCK_SIZE * 2 + 1; + + Matrix cacheMatrix1 = new SparseBlockDistributedMatrix(size, size2); + Matrix cacheMatrix2 = new SparseBlockDistributedMatrix(size2, size); + + for (int i = 0; i < size; i++) { + cacheMatrix1.setX(i, i, i); + cacheMatrix2.setX(i, i, i); + } + + Matrix res = cacheMatrix1.times(cacheMatrix2); + + for(int i = 0; i < size; i++) + for(int j = 0; j < size; j++) + if (i == j) + assertEquals(UNEXPECTED_VAL + " for "+ i +":"+ j, i * i, res.get(i, j), PRECISION); + else + assertEquals(UNEXPECTED_VAL + " for "+ i +":"+ j, 0, res.get(i, j), PRECISION); + } + + /** */ + private void initMtx(Matrix m) { + for (int i = 0; i < m.rowSize(); i++) + for (int j = 0; j < m.columnSize(); j++) + m.set(i, j, 1.0); + } + + /** Build key set for SparseBlockDistributedMatrix. */ + private Set buildKeySet(SparseBlockDistributedMatrix m){ + Set set = new HashSet<>(); + + BlockMatrixStorage storage = (BlockMatrixStorage)m.getStorage(); + + IgniteUuid uuid = storage.getUUID(); + + long maxBlock = (rows / 32 + (rows % 32 > 0 ? 1 : 0)) * (cols / 32 + (cols % 32 > 0 ? 1 : 0)); + + for (long i = 0; i < maxBlock; i++) + set.add(new BlockMatrixKey(i,uuid,null)); + + return set; + } +} diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java index a7cd6b5ac3e7e..3fec83ca4f24b 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java @@ -48,10 +48,10 @@ public class SparseDistributedMatrixTest extends GridCommonAbstractTest { /** Number of nodes in grid */ private static final int NODE_COUNT = 3; - /** Cache name. */ - private static final String CACHE_NAME = "test-cache"; /** Precision. */ private static final double PRECISION = 0.0; + /** */ + private static final int MATRIX_SIZE = 10; /** Grid instance. */ private Ignite ignite; /** Matrix rows */ @@ -90,8 +90,6 @@ public SparseDistributedMatrixTest() { /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { - ignite.destroyCache(CACHE_NAME); - if (cacheMatrix != null) { cacheMatrix.destroy(); cacheMatrix = null; @@ -166,7 +164,9 @@ public void testMath() { assertEquals(UNEXPECTED_VAL, cacheMatrix.rowSize() * cacheMatrix.columnSize(), cacheMatrix.sum(), PRECISION); } - /** */ + /** + * TODO: IGNITE-5102, wrong min/max, wait for fold/map fix + */ public void testMinMax() { IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); @@ -285,6 +285,28 @@ public void testLikeVector() { fail("UnsupportedOperationException expected."); } + /** */ + public void testMatrixTimes(){ + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + SparseDistributedMatrix cacheMatrix1 = new SparseDistributedMatrix(MATRIX_SIZE, MATRIX_SIZE, StorageConstants.ROW_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE); + SparseDistributedMatrix cacheMatrix2 = new SparseDistributedMatrix(MATRIX_SIZE, MATRIX_SIZE, StorageConstants.ROW_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE); + + for (int i = 0; i < MATRIX_SIZE; i++) { + cacheMatrix1.setX(i, i, i); + cacheMatrix2.setX(i, i, i); + } + + Matrix res = cacheMatrix1.times(cacheMatrix2); + + for(int i = 0; i < MATRIX_SIZE; i++) + for(int j = 0; j < MATRIX_SIZE; j++) + if (i == j) + assertEquals(UNEXPECTED_VAL, i * i, res.get(i, j), PRECISION); + else + assertEquals(UNEXPECTED_VAL, 0, res.get(i, j), PRECISION); + } + /** */ private void initMtx(Matrix m) { for (int i = 0; i < m.rowSize(); i++) From 48f29943efa9cbfc1e2c4068f7e16373dec2b0b9 Mon Sep 17 00:00:00 2001 From: Vitaliy Biryukov Date: Fri, 21 Jul 2017 15:29:23 +0300 Subject: [PATCH 008/547] IGNITE-3950 Deadlock when exchange starts with pending explicit lock --- .../cache/distributed/dht/IgniteCacheMultiTxLockSelfTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheMultiTxLockSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheMultiTxLockSelfTest.java index 6fd5dd3356cfb..11b0eea9a36af 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheMultiTxLockSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheMultiTxLockSelfTest.java @@ -117,8 +117,6 @@ public void testExplicitLockManyKeys() throws Exception { * @throws Exception If failed. */ public void testExplicitLockManyKeysWithClient() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-3950"); - checkExplicitLock(4, true); } From 199b954345f179851718acd131188506668cd4f3 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Fri, 21 Jul 2017 16:29:15 +0300 Subject: [PATCH 009/547] IGNITE-5752 Fixed updateSequence updating in GridDhtPartitionMap. - Fixes #2297. Signed-off-by: Alexey Goncharuk --- .../distributed/dht/preloader/GridDhtPartitionMap.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionMap.java index cfd4400d65a62..735ca1e1aaf7e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionMap.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionMap.java @@ -24,7 +24,6 @@ import java.util.Map; import java.util.Set; import java.util.UUID; - import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState; @@ -202,9 +201,13 @@ public long updateSequence() { * @return Old update sequence value. */ public long updateSequence(long updateSeq, AffinityTopologyVersion topVer) { + assert topVer.compareTo(top) >= 0 : "Invalid topology version [cur=" + top + ", new=" + topVer + "]"; + long old = this.updateSeq; - assert updateSeq >= old : "Invalid update sequence [cur=" + old + ", new=" + updateSeq + ']'; + // Overwrite update sequence without checking in case of greater topology version + if (topVer.compareTo(top) == 0) + assert updateSeq >= old : "Invalid update sequence [cur=" + old + ", new=" + updateSeq + ']'; this.updateSeq = updateSeq; From 6de0571c21ffdb77af7bb1d18e9659126d7f321b Mon Sep 17 00:00:00 2001 From: Ilya Lantukh Date: Fri, 21 Jul 2017 16:35:43 +0300 Subject: [PATCH 010/547] IGNITE-5772 - Fixed race between WAL segment rollover and a concurrent log. Closes #2313 --- .../wal/FileWriteAheadLogManager.java | 93 ++++++++++++------- 1 file changed, 61 insertions(+), 32 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index 897f903deda04..b655ddfe01991 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -319,7 +319,7 @@ protected String consistentId() { try { if (mode == WALMode.BACKGROUND) { if (currHnd != null) - currHnd.flush((FileWALPointer)null); + currHnd.flush((FileWALPointer)null, true); } if (currHnd != null) @@ -526,7 +526,7 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() { return; if (mode == WALMode.LOG_ONLY || forceFlush) { - cur.flushOrWait(filePtr); + cur.flushOrWait(filePtr, false); return; } @@ -535,7 +535,7 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() { if (filePtr != null && !cur.needFsync(filePtr)) return; - cur.fsync(filePtr); + cur.fsync(filePtr, false); } /** {@inheritDoc} */ @@ -1700,11 +1700,28 @@ private FileWriteHandle( this.maxSegmentSize = maxSegmentSize; this.serializer = serializer; - head.set(new FakeRecord(new FileWALPointer(idx, (int)pos, 0))); + head.set(new FakeRecord(new FileWALPointer(idx, (int)pos, 0), false)); written = pos; lastFsyncPos = pos; } + /** + * Checks if current head is a close fake record and returns {@code true} if so. + * + * @return {@code true} if current head is close record. + */ + private boolean stopped() { + return stopped(head.get()); + } + + /** + * @param record Record to check. + * @return {@code true} if the record is fake close record. + */ + private boolean stopped(WALRecord record) { + return record instanceof FakeRecord && ((FakeRecord)record).stop; + } + /** * @param rec Record to be added to record chain as new {@link #head} * @return Pointer or null if roll over to next segment is required or already started by other thread. @@ -1721,9 +1738,7 @@ private FileWriteHandle( long nextPos = nextPosition(h); - // It is important that we read `stop` after `head` in this loop for correct close, - // because otherwise we will have a race on the last flush in close. - if (nextPos + rec.size() >= maxSegmentSize || stop.get()) { + if (nextPos + rec.size() >= maxSegmentSize || stopped(h)) { // Can not write to this segment, need to switch to the next one. return null; } @@ -1731,7 +1746,7 @@ private FileWriteHandle( int newChainSize = h.chainSize() + rec.size(); if (newChainSize > tlbSize && !flushed) { - boolean res = h.previous() == null || flush(h); + boolean res = h.previous() == null || flush(h, false); if (rec.size() > tlbSize) flushed = res; @@ -1770,7 +1785,7 @@ private long nextPosition(WALRecord rec) { * @param ptr Pointer. * @throws IgniteCheckedException If failed. */ - private void flushOrWait(FileWALPointer ptr) throws IgniteCheckedException { + private void flushOrWait(FileWALPointer ptr, boolean stop) throws IgniteCheckedException { long expWritten; if (ptr != null) { @@ -1783,7 +1798,7 @@ private void flushOrWait(FileWALPointer ptr) throws IgniteCheckedException { else // We read head position before the flush because otherwise we can get wrong position. expWritten = recordOffset(head.get()); - if (flush(ptr)) + if (flush(ptr, stop)) return; // Spin-wait for a while before acquiring the lock. @@ -1810,18 +1825,20 @@ private void flushOrWait(FileWALPointer ptr) throws IgniteCheckedException { * @throws IgniteCheckedException If failed. * @throws StorageException If failed. */ - private boolean flush(FileWALPointer ptr) throws IgniteCheckedException, StorageException { + private boolean flush(FileWALPointer ptr, boolean stop) throws IgniteCheckedException, StorageException { if (ptr == null) { // Unconditional flush. for (; ; ) { WALRecord expHead = head.get(); if (expHead.previous() == null) { - assert expHead instanceof FakeRecord; + FakeRecord frHead = (FakeRecord)expHead; - return false; + if (frHead.stop == stop || frHead.stop || + head.compareAndSet(expHead, new FakeRecord(frHead.position(), stop))) + return false; } - if (flush(expHead)) + if (flush(expHead, stop)) return true; } } @@ -1835,7 +1852,7 @@ private boolean flush(FileWALPointer ptr) throws IgniteCheckedException, Storage if (chainBeginPosition(h) > ptr.fileOffset()) return false; - if (flush(h)) + if (flush(h, stop)) return true; // We are lucky. } } @@ -1853,17 +1870,18 @@ private long chainBeginPosition(WALRecord h) { * @throws IgniteCheckedException If failed. * @throws StorageException If failed. */ - private boolean flush(WALRecord expHead) throws StorageException, IgniteCheckedException { + private boolean flush(WALRecord expHead, boolean stop) throws StorageException, IgniteCheckedException { if (expHead.previous() == null) { - assert expHead instanceof FakeRecord; + FakeRecord frHead = (FakeRecord)expHead; - return false; + if (stop == frHead.stop) + return false; } // Fail-fast before CAS. checkEnvironment(); - if (!head.compareAndSet(expHead, new FakeRecord(new FileWALPointer(idx, (int)nextPosition(expHead), 0)))) + if (!head.compareAndSet(expHead, new FakeRecord(new FileWALPointer(idx, (int)nextPosition(expHead), 0), stop))) return false; // At this point we grabbed the piece of WAL chain. @@ -1976,7 +1994,7 @@ private FileWALPointer position() { * @param ptr Pointer to sync. * @throws StorageException If failed. */ - private void fsync(FileWALPointer ptr) throws StorageException, IgniteCheckedException { + private void fsync(FileWALPointer ptr, boolean stop) throws StorageException, IgniteCheckedException { lock.lock(); try { @@ -1984,7 +2002,7 @@ private void fsync(FileWALPointer ptr) throws StorageException, IgniteCheckedExc if (!needFsync(ptr)) return; - if (fsyncDelay > 0 && !stop.get()) { + if (fsyncDelay > 0 && !stopped()) { // Delay fsync to collect as many updates as possible: trade latency for throughput. U.await(fsync, fsyncDelay, TimeUnit.NANOSECONDS); @@ -1993,7 +2011,7 @@ private void fsync(FileWALPointer ptr) throws StorageException, IgniteCheckedExc } } - flushOrWait(ptr); + flushOrWait(ptr, stop); if (lastFsyncPos != written) { assert lastFsyncPos < written; // Fsync position must be behind. @@ -2031,13 +2049,14 @@ private void fsync(FileWALPointer ptr) throws StorageException, IgniteCheckedExc * @throws StorageException If failed. */ private boolean close(boolean rollOver) throws IgniteCheckedException, StorageException { - if (stop.compareAndSet(false, true)) { - // Here we can be sure that no other records will be added and this fsync will be the last. - if (mode == WALMode.DEFAULT) - fsync(null); - else - flushOrWait(null); + if (mode == WALMode.DEFAULT) + fsync(null, true); + else + flushOrWait(null, true); + + assert stopped() : "Segment is not closed after close flush: " + head.get(); + if (stop.compareAndSet(false, true)) { try { int switchSegmentRecSize = RecordV1Serializer.REC_TYPE_SIZE + RecordV1Serializer.FILE_WAL_POINTER_SIZE; @@ -2068,8 +2087,8 @@ private boolean close(boolean rollOver) throws IgniteCheckedException, StorageEx return true; } - - return false; + else + return false; } /** @@ -2271,17 +2290,27 @@ private static int recordOffset(WALRecord rec) { * Fake record is allowed to have no previous record. */ private static final class FakeRecord extends WALRecord { + /** */ + private final boolean stop; + /** * @param pos Position. */ - FakeRecord(FileWALPointer pos) { + FakeRecord(FileWALPointer pos, boolean stop) { position(pos); + + this.stop = stop; } /** {@inheritDoc} */ @Override public RecordType type() { return null; } + + /** {@inheritDoc} */ + @Override public FileWALPointer position() { + return (FileWALPointer) super.position(); + } } /** @@ -2492,7 +2521,7 @@ private void releaseWorkSegment(long absIdx) { private void doFlush() { final FileWriteHandle hnd = currentHandle(); try { - hnd.flush(hnd.head.get()); + hnd.flush(hnd.head.get(), false); } catch (Exception e) { U.warn(log, "Failed to flush WAL record queue", e); From c1a3b3744f89e27906621e62e9d73281791fcf30 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Fri, 21 Jul 2017 17:04:39 +0300 Subject: [PATCH 011/547] IGNITE-5786 .NET: Fix cache store session handling for cross-cache transactions This closes #2331 --- .../dotnet/PlatformDotNetCacheStore.java | 31 +++++++ .../Apache.Ignite.Core.Tests.csproj | 5 ++ .../Cache/Store/CacheStoreSessionTest.cs | 90 +++++++++++++------ .../Store/CacheStoreSessionTestCodeConfig.cs | 68 ++++++++++++++ .../CacheStoreSessionTestSharedFactory.cs | 48 ++++++++++ .../Cache/Store/CacheStoreTest.cs | 10 ++- .../cache-store-session-shared-factory.xml | 76 ++++++++++++++++ .../Cache/Store/cache-store-session.xml | 20 ++--- .../Impl/Cache/Store/CacheStoreInternal.cs | 14 ++- 9 files changed, 320 insertions(+), 42 deletions(-) create mode 100644 modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTestCodeConfig.cs create mode 100644 modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTestSharedFactory.cs create mode 100644 modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/Cache/Store/cache-store-session-shared-factory.xml diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/dotnet/PlatformDotNetCacheStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/dotnet/PlatformDotNetCacheStore.java index dd61a54e4bec8..471eb014c5065 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/dotnet/PlatformDotNetCacheStore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/dotnet/PlatformDotNetCacheStore.java @@ -47,6 +47,7 @@ import javax.cache.integration.CacheWriterException; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; /** @@ -90,6 +91,9 @@ public class PlatformDotNetCacheStore implements CacheStore, Platfor /** Key used to distinguish session deployment. */ private static final Object KEY_SES = new Object(); + /** Key to designate a set of stores that share current session. */ + private static final Object KEY_SES_STORES = new Object(); + /** */ @CacheStoreSessionResource private CacheStoreSession ses; @@ -337,6 +341,23 @@ public void setProperties(Map props) { writer.writeLong(session()); writer.writeString(ses.cacheName()); writer.writeBoolean(commit); + + // When multiple stores (caches) participate in a single transaction, + // they share a single session, but sessionEnd is called on each store. + // Same thing happens on platform side: session is shared; each store must be notified, + // then session should be closed. + Collection stores = (Collection) ses.properties().get(KEY_SES_STORES); + assert stores != null; + + stores.remove(ptr); + boolean last = stores.isEmpty(); + + writer.writeBoolean(last); + + if (last) { + // Session object has been released on platform side, remove marker. + ses.properties().remove(KEY_SES); + } } }, null); } @@ -415,6 +436,16 @@ private long session() throws IgniteCheckedException { ses.properties().put(KEY_SES, sesPtr); } + // Keep track of all stores that use current session (cross-cache tx uses single session for all caches). + Collection stores = (Collection) ses.properties().get(KEY_SES_STORES); + + if (stores == null) { + stores = new HashSet<>(); + ses.properties().put(KEY_SES_STORES, stores); + } + + stores.add(ptr); + return sesPtr; } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj index 90b79705e0aaa..e4f65bc85b3db 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj @@ -89,6 +89,8 @@ + + @@ -314,6 +316,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTest.cs index 315e285113758..818948c337a43 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTest.cs @@ -28,16 +28,13 @@ namespace Apache.Ignite.Core.Tests.Cache.Store /// /// Tests for store session. /// - public sealed class CacheStoreSessionTest + public class CacheStoreSessionTest { - /** Grid name. */ - private const string IgniteName = "grid"; - /** Cache 1 name. */ - private const string Cache1 = "cache1"; + protected const string Cache1 = "cache1"; /** Cache 2 name. */ - private const string Cache2 = "cache2"; + protected const string Cache2 = "cache2"; /** Operations. */ private static ConcurrentBag> _dumps; @@ -48,11 +45,26 @@ public sealed class CacheStoreSessionTest [TestFixtureSetUp] public void BeforeTests() { - Ignition.Start(new IgniteConfiguration(TestUtils.GetTestConfiguration()) + Ignition.Start(GetIgniteConfiguration()); + } + + /// + /// Gets the ignite configuration. + /// + protected virtual IgniteConfiguration GetIgniteConfiguration() + { + return new IgniteConfiguration(TestUtils.GetTestConfiguration()) { - IgniteInstanceName = IgniteName, SpringConfigUrl = @"config\cache\store\cache-store-session.xml" - }); + }; + } + + /// + /// Gets the store count. + /// + protected virtual int StoreCount + { + get { return 2; } } /// @@ -61,21 +73,29 @@ public void BeforeTests() [TestFixtureTearDown] public void AfterTests() { - Ignition.StopAll(true); + try + { + TestUtils.AssertHandleRegistryHasItems(Ignition.GetIgnite(), 2, 1000); + } + finally + { + Ignition.StopAll(true); + } } /// /// Test basic session API. /// [Test] + [Timeout(30000)] public void TestSession() { _dumps = new ConcurrentBag>(); - var ignite = Ignition.GetIgnite(IgniteName); + var ignite = Ignition.GetIgnite(); - var cache1 = Ignition.GetIgnite(IgniteName).GetCache(Cache1); - var cache2 = Ignition.GetIgnite(IgniteName).GetCache(Cache2); + var cache1 = ignite.GetCache(Cache1); + var cache2 = ignite.GetCache(Cache2); // 1. Test rollback. using (var tx = ignite.GetTransactions().TxStart()) @@ -86,11 +106,15 @@ public void TestSession() tx.Rollback(); } - Assert.AreEqual(1, _dumps.Count); - var ops = _dumps.First(); - Assert.AreEqual(1, ops.Count); + // SessionEnd is called once per store instance. + Assert.AreEqual(StoreCount, _dumps.Count); - Assert.AreEqual(1, ops.Count(op => op.Type == OperationType.SesEnd && !op.Commit)); + foreach (var ops in _dumps) + { + var op = ops.Single(); + Assert.AreEqual(OperationType.SesEnd, op.Type); + Assert.IsFalse(op.Commit); + } _dumps = new ConcurrentBag>(); @@ -103,13 +127,17 @@ public void TestSession() tx.Commit(); } - Assert.AreEqual(1, _dumps.Count); - ops = _dumps.First(); - Assert.AreEqual(3, ops.Count); + Assert.AreEqual(StoreCount, _dumps.Count); - Assert.AreEqual(1, ops.Count(op => op.Type == OperationType.Write && Cache1.Equals(op.CacheName) && 1.Equals(op.Key) && 1.Equals(op.Value))); - Assert.AreEqual(1, ops.Count(op => op.Type == OperationType.Write && Cache2.Equals(op.CacheName) && 2.Equals(op.Key) && 2.Equals(op.Value))); - Assert.AreEqual(1, ops.Count(op => op.Type == OperationType.SesEnd && op.Commit)); + foreach (var ops in _dumps) + { + Assert.AreEqual(2 + StoreCount, ops.Count); + Assert.AreEqual(1, ops.Count(op => op.Type == OperationType.Write + && Cache1 == op.CacheName && 1 == op.Key && 1 == op.Value)); + Assert.AreEqual(1, ops.Count(op => op.Type == OperationType.Write + && Cache2 == op.CacheName && 2 == op.Key && 2 == op.Value)); + Assert.AreEqual(StoreCount, ops.Count(op => op.Type == OperationType.SesEnd && op.Commit)); + } _dumps = new ConcurrentBag>(); @@ -122,13 +150,17 @@ public void TestSession() tx.Commit(); } - Assert.AreEqual(1, _dumps.Count); - ops = _dumps.First(); - Assert.AreEqual(3, ops.Count); + Assert.AreEqual(StoreCount, _dumps.Count); + foreach (var ops in _dumps) + { + Assert.AreEqual(2 + StoreCount, ops.Count); - Assert.AreEqual(1, ops.Count(op => op.Type == OperationType.Delete && Cache1.Equals(op.CacheName) && 1.Equals(op.Key))); - Assert.AreEqual(1, ops.Count(op => op.Type == OperationType.Delete && Cache2.Equals(op.CacheName) && 2.Equals(op.Key))); - Assert.AreEqual(1, ops.Count(op => op.Type == OperationType.SesEnd && op.Commit)); + Assert.AreEqual(1, ops.Count(op => op.Type == OperationType.Delete + && Cache1 == op.CacheName && 1 == op.Key)); + Assert.AreEqual(1, ops.Count(op => op.Type == OperationType.Delete + && Cache2 == op.CacheName && 2 == op.Key)); + Assert.AreEqual(StoreCount, ops.Count(op => op.Type == OperationType.SesEnd && op.Commit)); + } } /// diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTestCodeConfig.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTestCodeConfig.cs new file mode 100644 index 0000000000000..0b5f474d7f5d0 --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTestCodeConfig.cs @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Tests.Cache.Store +{ + using Apache.Ignite.Core.Cache.Configuration; + using Apache.Ignite.Core.Cache.Store; + using Apache.Ignite.Core.Common; + using NUnit.Framework; + + /// + /// Tests store session with programmatic configuration (uses different store factory on Java side). + /// + [TestFixture] + public class CacheStoreSessionTestCodeConfig : CacheStoreSessionTest + { + /** */ + protected override IgniteConfiguration GetIgniteConfiguration() + { + return new IgniteConfiguration(TestUtils.GetTestConfiguration()) + { + CacheConfiguration = new[] + { + new CacheConfiguration(Cache1) + { + AtomicityMode = CacheAtomicityMode.Transactional, + ReadThrough = true, + WriteThrough = true, + CacheStoreFactory = new StoreFactory() + }, + new CacheConfiguration(Cache2) + { + AtomicityMode = CacheAtomicityMode.Transactional, + ReadThrough = true, + WriteThrough = true, + CacheStoreFactory = new StoreFactory() + } + } + }; + } + + /// + /// Store factory. + /// + private class StoreFactory : IFactory + { + /** */ + public ICacheStore CreateInstance() + { + return new Store(); + } + } + } +} \ No newline at end of file diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTestSharedFactory.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTestSharedFactory.cs new file mode 100644 index 0000000000000..2af591570cd23 --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreSessionTestSharedFactory.cs @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Tests.Cache.Store +{ + using NUnit.Framework; + + /// + /// Session test with shared PlatformDotNetCacheStoreFactory, + /// which causes the same store insance to be used for both caches. + /// + [TestFixture] + public class CacheStoreSessionTestSharedFactory : CacheStoreSessionTest + { + /** */ + protected override IgniteConfiguration GetIgniteConfiguration() + { + return new IgniteConfiguration(TestUtils.GetTestConfiguration()) + { + SpringConfigUrl = @"config\cache\store\cache-store-session-shared-factory.xml" + }; + } + + /** */ + protected override int StoreCount + { + get + { + // Shared PlatformDotNetCacheStoreFactory results in a single store instance. + return 1; + } + } + } +} \ No newline at end of file diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreTest.cs index e05f4bded7796..d3e4ab6fa7486 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreTest.cs @@ -64,7 +64,15 @@ public virtual void BeforeTests() [TestFixtureTearDown] public void AfterTests() { - Ignition.StopAll(true); + try + { + // 3 stores are expected in HandleRegistry. + TestUtils.AssertHandleRegistryHasItems(Ignition.GetIgnite(), 3, 1000); + } + finally + { + Ignition.StopAll(true); + } } /// diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/Cache/Store/cache-store-session-shared-factory.xml b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/Cache/Store/cache-store-session-shared-factory.xml new file mode 100644 index 0000000000000..05515c4f75079 --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/Cache/Store/cache-store-session-shared-factory.xml @@ -0,0 +1,76 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 127.0.0.1:47500 + + + + + + + + + diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/Cache/Store/cache-store-session.xml b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/Cache/Store/cache-store-session.xml index 3cc9efae90c6e..14dc78eef965c 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/Cache/Store/cache-store-session.xml +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/Cache/Store/cache-store-session.xml @@ -25,18 +25,10 @@ http://www.springframework.org/schema/beans/spring-beans.xsd http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd"> - - - - - - - - @@ -46,7 +38,11 @@ - + + + + + @@ -56,7 +52,11 @@ - + + + + + diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Store/CacheStoreInternal.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Store/CacheStoreInternal.cs index f147579d6e39d..df4c1ae5d3fa2 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Store/CacheStoreInternal.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Store/CacheStoreInternal.cs @@ -111,6 +111,8 @@ public int Invoke(IBinaryStream stream, Ignite grid) CacheStoreSession ses = grid.HandleRegistry.Get(sesId, true); + // Session cache name may change in cross-cache transaction. + // Single session is used for all stores in cross-cache transactions. ses.CacheName = rawReader.ReadString(); _sesProxy.SetSession(ses); @@ -223,11 +225,19 @@ public int Invoke(IBinaryStream stream, Ignite grid) break; case OpSesEnd: - grid.HandleRegistry.Release(sesId); + { + var commit = rawReader.ReadBoolean(); + var last = rawReader.ReadBoolean(); - _store.SessionEnd(rawReader.ReadBoolean()); + if (last) + { + grid.HandleRegistry.Release(sesId); + } + + _store.SessionEnd(commit); break; + } default: throw new IgniteException("Invalid operation type: " + opType); From 6f749bf4bebb135250a3f99923ed87b4b7d0c29f Mon Sep 17 00:00:00 2001 From: Sergey Chugunov Date: Fri, 21 Jul 2017 17:59:10 +0300 Subject: [PATCH 012/547] IGNITE-5067 - Fixed absolute swap file path handling for memory policy configuration. Fixes #1867 --- .../IgniteCacheDatabaseSharedManager.java | 22 ++- .../SwapPathConstructionSelfTest.java | 157 ++++++++++++++++++ .../testsuites/IgniteBasicTestSuite.java | 3 + 3 files changed, 174 insertions(+), 8 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/database/SwapPathConstructionSelfTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index eec3b85c8fb7c..e07c51ea76a28 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -25,7 +25,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import javax.management.JMException; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.MemoryMetrics; @@ -204,8 +203,9 @@ private void startMemoryPolicies() { /** * @param memCfg Database config. + * @throws IgniteCheckedException If failed to initialize swap path. */ - protected void initPageMemoryPolicies(MemoryConfiguration memCfg) { + protected void initPageMemoryPolicies(MemoryConfiguration memCfg) throws IgniteCheckedException { MemoryPolicyConfiguration[] memPlcsCfgs = memCfg.getMemoryPolicies(); if (memPlcsCfgs == null) { @@ -261,12 +261,13 @@ protected void initPageMemoryPolicies(MemoryConfiguration memCfg) { * @param memCfg Database config. * @param memPlcCfg Memory policy config. * @param memPlcName Memory policy name. + * @throws IgniteCheckedException If failed to initialize swap path. */ private void addMemoryPolicy( MemoryConfiguration memCfg, MemoryPolicyConfiguration memPlcCfg, String memPlcName - ) { + ) throws IgniteCheckedException { String dfltMemPlcName = memCfg.getDefaultMemoryPolicyName(); if (dfltMemPlcName == null) @@ -844,12 +845,14 @@ public void ensureFreeSpace(MemoryPolicy memPlc) throws IgniteCheckedException { * @param plcCfg memory policy with PageMemory specific parameters. * @param memMetrics {@link MemoryMetrics} object to collect memory usage metrics. * @return Memory policy instance. + * + * @throws IgniteCheckedException If failed to initialize swap path. */ private MemoryPolicy initMemory( MemoryConfiguration memCfg, MemoryPolicyConfiguration plcCfg, MemoryMetricsImpl memMetrics - ) { + ) throws IgniteCheckedException { File allocPath = buildAllocPath(plcCfg); DirectMemoryProvider memProvider = allocPath == null ? @@ -892,8 +895,10 @@ private PageEvictionTracker createPageEvictionTracker(MemoryPolicyConfiguration * Builds allocation path for memory mapped file to be used with PageMemory. * * @param plc MemoryPolicyConfiguration. + * + * @throws IgniteCheckedException If resolving swap directory fails. */ - @Nullable protected File buildAllocPath(MemoryPolicyConfiguration plc) { + @Nullable protected File buildAllocPath(MemoryPolicyConfiguration plc) throws IgniteCheckedException { String path = plc.getSwapFilePath(); if (path == null) @@ -938,13 +943,14 @@ protected PageMemory createPageMemory( * @param path Path to the working directory. * @param consId Consistent ID of the local node. * @return DB storage path. + * + * @throws IgniteCheckedException If resolving swap directory fails. */ - protected File buildPath(String path, String consId) { + protected File buildPath(String path, String consId) throws IgniteCheckedException { String igniteHomeStr = U.getIgniteHome(); - File igniteHome = igniteHomeStr != null ? new File(igniteHomeStr) : null; + File workDir = igniteHomeStr == null ? new File(path) : U.resolveWorkDirectory(igniteHomeStr, path, false); - File workDir = igniteHome == null ? new File(path) : new File(igniteHome, path); return new File(workDir, consId); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/SwapPathConstructionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/SwapPathConstructionSelfTest.java new file mode 100644 index 0000000000000..53e5daf81f96d --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/SwapPathConstructionSelfTest.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.database; + +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.MemoryConfiguration; +import org.apache.ignite.configuration.MemoryPolicyConfiguration; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.pagemem.PageMemory; +import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; +import org.apache.ignite.internal.processors.cache.persistence.MemoryPolicy; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * Test verifies correct construction of swap file path {@link MemoryPolicyConfiguration#setSwapFilePath(String)} + * when absolute or relative paths are provided via configuration. + */ +public class SwapPathConstructionSelfTest extends GridCommonAbstractTest { + /** */ + private MemoryConfiguration memCfg; + + /** */ + private static final String RELATIVE_SWAP_PATH = "relSwapPath"; + + /** */ + private static final String ABSOLUTE_SWAP_PATH = "absoluteSwapPath"; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setMemoryConfiguration(memCfg); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + cleanUpSwapDir(); + } + + /** + * Cleans up swap files and directories after test. + */ + private void cleanUpSwapDir() { + Path relDir = Paths.get(U.getIgniteHome(), RELATIVE_SWAP_PATH); + + deleteRecursively(relDir.toFile()); + + Path absDir = Paths.get(getTmpDir(), ABSOLUTE_SWAP_PATH); + + deleteRecursively(absDir.toFile()); + } + + /** + * Verifies relative swap file path construction. Directory with swap files is cleaned up during after-test phase. + */ + public void testRelativeSwapFilePath() throws Exception { + memCfg = createMemoryConfiguration(true); + + IgniteEx ignite = startGrid(0); + + String allocPath = extractDefaultPageMemoryAllocPath(ignite.context()); + + assertNotNull(allocPath); + + assertTrue(allocPath.contains(Paths.get(U.getIgniteHome(), RELATIVE_SWAP_PATH).toString())); + } + + /** + * Verifies absolute swap file path construction. System tmp directory is used to allocate swap files, + * so no clean up is needed. + */ + public void testAbsoluteSwapFilePath() throws Exception { + memCfg = createMemoryConfiguration(false); + + IgniteEx ignite = startGrid(0); + + String allocPath = extractDefaultPageMemoryAllocPath(ignite.context()); + + assertNotNull(allocPath); + + String expectedPath = Paths.get(getTmpDir(), ABSOLUTE_SWAP_PATH).toString(); + + assertTrue("Expected path: " + + expectedPath + + "; actual path: " + + allocPath, + allocPath.startsWith(expectedPath)); + } + + /** + * @param context Context. + */ + private String extractDefaultPageMemoryAllocPath(GridKernalContext context) { + IgniteCacheDatabaseSharedManager dbMgr = context.cache().context().database(); + + Map memPlcMap = U.field(dbMgr, "memPlcMap"); + + PageMemory pageMem = memPlcMap.get("default").pageMemory(); + + Object memProvider = U.field(pageMem, "directMemoryProvider"); + + return ((File) U.field(memProvider, "allocationPath")).getAbsolutePath(); + } + + /** + * @param isRelativePath flag is set to {@code true} if relative path should be used for memory policy configuration. + */ + private MemoryConfiguration createMemoryConfiguration(boolean isRelativePath) { + MemoryConfiguration memCfg = new MemoryConfiguration(); + + MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration(); + + memPlcCfg.setName("default"); + memPlcCfg.setMaxSize(20 * 1024 * 1024); + + if (isRelativePath) + memPlcCfg.setSwapFilePath(RELATIVE_SWAP_PATH); + else + memPlcCfg.setSwapFilePath(Paths.get(getTmpDir(), ABSOLUTE_SWAP_PATH).toString()); + + memCfg.setMemoryPolicies(memPlcCfg); + + return memCfg; + } + + /** + * + */ + private String getTmpDir() { + return System.getProperty("java.io.tmpdir"); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java index d79e8683d6bdd..2ec2c74d28f90 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java @@ -20,6 +20,8 @@ import java.util.Set; import junit.framework.TestSuite; import org.apache.ignite.GridSuppressedExceptionSelfTest; +import org.apache.ignite.internal.processors.database.SwapPathConstructionSelfTest; +import org.apache.ignite.util.AttributeNodeFilterSelfTest; import org.apache.ignite.internal.ClusterGroupHostsSelfTest; import org.apache.ignite.internal.ClusterGroupSelfTest; import org.apache.ignite.internal.GridFailFastNodeFailureDetectionSelfTest; @@ -173,6 +175,7 @@ public static TestSuite suite(@Nullable final Set ignoredTests) throws Ex suite.addTestSuite(MetadataStorageSelfTest.class); suite.addTestSuite(FreeListImplSelfTest.class); suite.addTestSuite(MemoryMetricsSelfTest.class); + suite.addTestSuite(SwapPathConstructionSelfTest.class); suite.addTestSuite(IgniteMarshallerCacheFSRestoreTest.class); suite.addTestSuite(IgniteMarshallerCacheClassNameConflictTest.class); From 95f4abc709bcf2a4534f9631b3ecea8266cfc485 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Fri, 21 Jul 2017 18:03:07 +0300 Subject: [PATCH 013/547] Fixed checking full map on updates. --- .../distributed/dht/GridClientPartitionTopology.java | 9 +++++++-- .../distributed/dht/GridDhtPartitionTopologyImpl.java | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java index f4ed517975155..6ff572b189f2c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java @@ -628,8 +628,13 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD } } - for (GridDhtPartitionMap part : partMap.values()) - fullMapUpdated |= !node2part.containsKey(part); + // Check that we have new nodes. + for (GridDhtPartitionMap part : partMap.values()) { + if (fullMapUpdated) + break; + + fullMapUpdated = !node2part.containsKey(part.nodeId()); + } // Remove entry if node left. for (Iterator it = partMap.keySet().iterator(); it.hasNext(); ) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java index 601da1b4ea2f4..0ca291dd95dc7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java @@ -1154,8 +1154,13 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD } } - for (GridDhtPartitionMap part : partMap.values()) - fullMapUpdated |= !node2part.containsKey(part); + // Check that we have new nodes. + for (GridDhtPartitionMap part : partMap.values()) { + if (fullMapUpdated) + break; + + fullMapUpdated = !node2part.containsKey(part.nodeId()); + } // Remove entry if node left. for (Iterator it = partMap.keySet().iterator(); it.hasNext(); ) { From abc6e46fb9ec1c5ee228c01943c516ba0f602456 Mon Sep 17 00:00:00 2001 From: Andrey Novikov Date: Mon, 24 Jul 2017 10:54:01 +0700 Subject: [PATCH 014/547] IGNITE-5754 Web Console agent: Use POST instead of GET for requests. --- .../web-console/backend/app/agentSocket.js | 4 +- .../backend/app/browsersHandler.js | 2 +- .../console/agent/handlers/RestListener.java | 7 +-- .../console/agent/rest/RestExecutor.java | 49 +++++++------------ 4 files changed, 22 insertions(+), 40 deletions(-) diff --git a/modules/web-console/backend/app/agentSocket.js b/modules/web-console/backend/app/agentSocket.js index db1deaa7322d6..08533392968d7 100644 --- a/modules/web-console/backend/app/agentSocket.js +++ b/modules/web-console/backend/app/agentSocket.js @@ -126,7 +126,7 @@ module.exports.factory = function(_) { * Send event to agent. * * @param {String} event - Event name. - * @param {Array.?} args - Transmitted arguments. + * @param {Object?} args - Transmitted arguments. * @returns {Promise} */ emitEvent(event, ...args) { @@ -202,7 +202,7 @@ module.exports.factory = function(_) { params[`p${idx + 1}`] = args[idx]; }); - return this.emitEvent('node:rest', {uri: 'ignite', demo, params, method: 'GET'}) + return this.emitEvent('node:rest', {uri: 'ignite', demo, params}) .then(this.restResultParse); } diff --git a/modules/web-console/backend/app/browsersHandler.js b/modules/web-console/backend/app/browsersHandler.js index c4ea5324fa8c6..4fb5088ee5ef6 100644 --- a/modules/web-console/backend/app/browsersHandler.js +++ b/modules/web-console/backend/app/browsersHandler.js @@ -179,7 +179,7 @@ module.exports.factory = (_, socketio, configure, errors, mongo) => { */ executeOnNode(agent, demo, params) { return agent - .then((agentSock) => agentSock.emitEvent('node:rest', {uri: 'ignite', demo, params, method: 'GET'})) + .then((agentSock) => agentSock.emitEvent('node:rest', {uri: 'ignite', demo, params})) .then((res) => { if (res.status === 0) return JSON.parse(res.data); diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java index c70514d3d8b76..8855060807e3b 100644 --- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java +++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java @@ -61,11 +61,6 @@ public RestListener(RestExecutor restExecutor) { boolean demo = (boolean)args.get("demo"); - if (!args.containsKey("method")) - throw new IllegalArgumentException("Missing method in arguments: " + args); - - String mtd = args.get("method").toString(); - Map headers = null; if (args.containsKey("headers")) @@ -76,6 +71,6 @@ public RestListener(RestExecutor restExecutor) { if (args.containsKey("body")) body = args.get("body").toString(); - return restExecutor.execute(demo, path, params, mtd, headers, body); + return restExecutor.execute(demo, path, params, headers, body); } } diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java index 3936374110868..03eca4e557173 100644 --- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java +++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java @@ -90,7 +90,7 @@ public void stop() { /** */ private RestResult sendRequest(boolean demo, String path, Map params, - String mtd, Map headers, String body) throws IOException { + Map headers, String body) throws IOException { if (demo && AgentClusterDemo.getDemoUrl() == null) { try { AgentClusterDemo.tryStart().await(); @@ -120,36 +120,24 @@ private RestResult sendRequest(boolean demo, String path, Map pa reqBuilder.addHeader(entry.getKey(), entry.getValue().toString()); } - if ("GET".equalsIgnoreCase(mtd)) { + if (body != null) { + MediaType contentType = MediaType.parse("text/plain"); + + reqBuilder.post(RequestBody.create(contentType, body)); + } + else { + FormBody.Builder formBody = new FormBody.Builder(); + if (params != null) { for (Map.Entry entry : params.entrySet()) { if (entry.getValue() != null) - urlBuilder.addQueryParameter(entry.getKey(), entry.getValue().toString()); + formBody.add(entry.getKey(), entry.getValue().toString()); } } - } - else if ("POST".equalsIgnoreCase(mtd)) { - if (body != null) { - MediaType contentType = MediaType.parse("text/plain"); - - reqBuilder.post(RequestBody.create(contentType, body)); - } - else { - FormBody.Builder formBody = new FormBody.Builder(); - - if (params != null) { - for (Map.Entry entry : params.entrySet()) { - if (entry.getValue() != null) - formBody.add(entry.getKey(), entry.getValue().toString()); - } - } - reqBuilder.post(formBody.build()); - } + reqBuilder.post(formBody.build()); } - else - throw new IllegalArgumentException("Unknown HTTP-method: " + mtd); - + reqBuilder.url(urlBuilder.build()); try (Response resp = httpClient.newCall(reqBuilder.build()).execute()) { @@ -180,7 +168,7 @@ else if ("POST".equalsIgnoreCase(mtd)) { "Please ensure that nodes have [ignite-rest-http] module in classpath " + "(was copied from libs/optional to libs folder)."); - throw new ConnectException("Failed connect to node and execute REST command [url=" + urlBuilder + "]"); + throw new ConnectException("Failed connect to node and execute REST command [url=" + urlBuilder + ", parameters=" + params + "]"); } } @@ -188,21 +176,20 @@ else if ("POST".equalsIgnoreCase(mtd)) { * @param demo Is demo node request. * @param path Path segment. * @param params Params. - * @param mtd Method. * @param headers Headers. * @param body Body. */ public RestResult execute(boolean demo, String path, Map params, - String mtd, Map headers, String body) { + Map headers, String body) { if (log.isDebugEnabled()) - log.debug("Start execute REST command [method=" + mtd + ", uri=/" + (path == null ? "" : path) + + log.debug("Start execute REST command [uri=/" + (path == null ? "" : path) + ", parameters=" + params + "]"); try { - return sendRequest(demo, path, params, mtd, headers, body); + return sendRequest(demo, path, params, headers, body); } catch (Exception e) { - U.error(log, "Failed to execute REST command [method=" + mtd + ", uri=/" + (path == null ? "" : path) + + U.error(log, "Failed to execute REST command [uri=/" + (path == null ? "" : path) + ", parameters=" + params + "]", e); return RestResult.fail(404, e.getMessage()); @@ -219,6 +206,6 @@ public RestResult topology(boolean demo, boolean full) throws IOException { params.put("attr", true); params.put("mtr", full); - return sendRequest(demo, "ignite", params, "GET", null, null); + return sendRequest(demo, "ignite", params, null, null); } } From 0d2992c502d54f14b71afbc98529770f49f32345 Mon Sep 17 00:00:00 2001 From: Andrey Novikov Date: Mon, 24 Jul 2017 14:30:05 +0700 Subject: [PATCH 015/547] Web Console: Fixed UI for header menu. --- .../components/web-console-header/style.scss | 6 +-- .../states/configuration/clusters/memory.pug | 8 ++-- .../frontend/app/primitives/panel/index.scss | 2 +- .../frontend/views/includes/header-left.pug | 41 ++++++++++--------- 4 files changed, 29 insertions(+), 28 deletions(-) diff --git a/modules/web-console/frontend/app/components/web-console-header/style.scss b/modules/web-console/frontend/app/components/web-console-header/style.scss index 376edd9b12a8b..5cc23551d02e9 100644 --- a/modules/web-console/frontend/app/components/web-console-header/style.scss +++ b/modules/web-console/frontend/app/components/web-console-header/style.scss @@ -18,7 +18,7 @@ web-console-header { @import "./../../../public/stylesheets/variables.scss"; - $nav-item-margin: 42px; + $nav-item-margin: 40px; $bottom-border-width: 4px; display: block; @@ -51,7 +51,7 @@ web-console-header { } &.wch-slot-left { - margin-left: 105px; + margin-left: 80px; } &.wch-slot-right { @@ -73,7 +73,7 @@ web-console-header { } .wch-nav-item, - a.wch-nav-item { + .wch-nav-item > a { cursor: pointer; white-space: nowrap; diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/memory.pug b/modules/web-console/frontend/app/modules/states/configuration/clusters/memory.pug index 0ba1ab4825f46..a09feddfa2207 100644 --- a/modules/web-console/frontend/app/modules/states/configuration/clusters/memory.pug +++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/memory.pug @@ -58,12 +58,12 @@ include /app/helpers/jade/mixins | Memory policies configuration .group-content .details-row - +text('Default region name:', model + '.defaultMemoryPolicyName', '"defaultMemoryPolicyName"', - 'false', 'default', 'Name of the memory policy that defines the default memory region') + +text('Default memory policy name:', model + '.defaultMemoryPolicyName', '"defaultMemoryPolicyName"', + 'false', 'default', 'Name of a memory policy to be used as default one') .details-row(ng-hide='(' + model + '.defaultMemoryPolicyName || "default") !== "default"') - +number('Default region size:', model + '.defaultMemoryPolicySize', '"defaultMemoryPolicySize"', + +number('Default memory policy size:', model + '.defaultMemoryPolicySize', '"defaultMemoryPolicySize"', 'true', '0.8 * totalMemoryAvailable', '10485760', - 'Size of the memory policy that defines the default memory region') + 'Specify desired size of default memory policy without having to use more verbose syntax of MemoryPolicyConfiguration elements') .details-row(ng-init='memoryPoliciesTbl={type: "memoryPolicies", model: "memoryPolicies", focusId: "name", ui: "memory-policies-table"}') +ignite-form-group() ignite-form-field-label diff --git a/modules/web-console/frontend/app/primitives/panel/index.scss b/modules/web-console/frontend/app/primitives/panel/index.scss index 38d18dd5f6692..56cecfe57646f 100644 --- a/modules/web-console/frontend/app/primitives/panel/index.scss +++ b/modules/web-console/frontend/app/primitives/panel/index.scss @@ -31,7 +31,7 @@ padding: 22px 20px; background-color: initial; - border-bottom: 1px solid $panel-default-border; + border-bottom: 1px solid $ignite-brand-primary; &:hover { text-decoration: none; diff --git a/modules/web-console/frontend/views/includes/header-left.pug b/modules/web-console/frontend/views/includes/header-left.pug index e9db53be8294f..6b7fe6aa8d6aa 100644 --- a/modules/web-console/frontend/views/includes/header-left.pug +++ b/modules/web-console/frontend/views/includes/header-left.pug @@ -14,15 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. -a.wch-nav-item( - ui-sref='base.configuration.tabs' - ui-sref-active='active' -) - | Configure +.wch-nav-item + a(ui-sref='base.configuration.tabs' ui-sref-active='active') + | Configure .wch-nav-item(ng-controller='notebookController') - div(ng-if='IgniteDemoMode' ui-sref='base.sql.demo' ng-class='{active: $state.includes("base.sql")}') - span Queries + a(ng-if='IgniteDemoMode' ui-sref='base.sql.demo' ui-sref-active='active') + | Queries div(ng-if='!IgniteDemoMode') div(ng-if='!notebooks.length' ng-class='{active: $state.includes("base.sql")}') @@ -41,16 +39,19 @@ a.wch-nav-item( span Queries span.caret -.wch-nav-item(ignite-navbar) - div(ng-click='$event.stopPropagation()' - ng-class='{active: $state.includes("base.monitoring")}' - ng-repeat='item in navbar.items' - bs-dropdown='item.children' - data-placement='bottom-left' - data-trigger='hover focus' - data-container='self' - aria-haspopup='true' - aria-expanded='false' - ) - span {{::item.text}} - span.caret +.wch-content(ignite-navbar) + .wch-nav-item(ng-repeat='item in navbar.items') + div(ng-if='item.children' ng-click='$event.stopPropagation()' + ng-class='{active: $state.includes(item.sref)}' + bs-dropdown='item.children' + data-placement='bottom-left' + data-trigger='hover focus' + data-container='self' + aria-haspopup='true' + aria-expanded='false' + ) + span {{::item.text}} + span.caret + + a(ng-if='!item.children' ui-sref='{{item.sref}}' ui-sref-active='active') + | {{::item.text}} \ No newline at end of file From aeb9336b3b161ddfff73f17e41cd453409b84a16 Mon Sep 17 00:00:00 2001 From: sboikov Date: Mon, 24 Jul 2017 11:47:16 +0300 Subject: [PATCH 016/547] Test for cache partitions state, fix for client cache start. --- .../cache/CacheAffinitySharedManager.java | 51 ++- .../dht/GridClientPartitionTopology.java | 7 +- .../dht/GridDhtPartitionTopology.java | 12 +- .../dht/GridDhtPartitionTopologyImpl.java | 45 +- .../GridDhtPartitionsExchangeFuture.java | 120 +++-- .../GridCacheDatabaseSharedManager.java | 6 +- .../CacheLateAffinityAssignmentTest.java | 36 +- .../distributed/CachePartitionStateTest.java | 410 ++++++++++++++++++ .../TestCacheNodeExcludingFilter.java | 53 +++ .../db/IgnitePdsCacheRestoreTest.java | 208 +++++++++ .../testsuites/IgniteCacheTestSuite6.java | 38 ++ .../ignite/testsuites/IgnitePdsTestSuite.java | 3 + 12 files changed, 863 insertions(+), 126 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionStateTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/TestCacheNodeExcludingFilter.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheRestoreTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java index 79ab183485600..f519b4e8fbff6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java @@ -517,6 +517,16 @@ else if (!fetchFuts.containsKey(grp.groupId())) { } } + for (DynamicCacheDescriptor desc : startDescs) { + if (desc.cacheConfiguration().getCacheMode() != LOCAL) { + CacheGroupContext grp = cctx.cache().cacheGroup(desc.groupId()); + + assert grp != null; + + grp.topology().onExchangeDone(grp.affinity().cachedAffinity(topVer), true); + } + } + cctx.cache().initCacheProxies(topVer, null); cctx.cache().completeClientCacheChangeFuture(msg.requestId(), null); @@ -1298,6 +1308,19 @@ private String groupNames(Collection grpIds) { return names.toString(); } + /** + * @param grpId Group ID. + * @return Group name for debug purpose. + */ + private String debugGroupName(int grpId) { + CacheGroupDescriptor desc = caches.group(grpId); + + if (desc != null) + return desc.cacheOrGroupName(); + else + return "Unknown group: " + grpId; + } + /** * @param fut Exchange future. * @throws IgniteCheckedException If failed. @@ -1396,19 +1419,31 @@ private GridDhtAffinityAssignmentResponse fetchAffinity(AffinityTopologyVersion * Called on exchange initiated by server node leave. * * @param fut Exchange future. + * @param crd Coordinator flag. * @throws IgniteCheckedException If failed. * @return {@code True} if affinity should be assigned by coordinator. */ - public boolean onServerLeft(final GridDhtPartitionsExchangeFuture fut) throws IgniteCheckedException { + public boolean onServerLeft(final GridDhtPartitionsExchangeFuture fut, boolean crd) throws IgniteCheckedException { ClusterNode leftNode = fut.discoveryEvent().eventNode(); assert !leftNode.isClient() : leftNode; - for (CacheGroupContext grp : cctx.cache().cacheGroups()) { - if (grp.isLocal()) - continue; + if (crd) { + // Need initialize CacheGroupHolders if this node become coordinator on this exchange. + forAllRegisteredCacheGroups(new IgniteInClosureX() { + @Override public void applyx(CacheGroupDescriptor desc) throws IgniteCheckedException { + CacheGroupHolder cache = groupHolder(fut.topologyVersion(), desc); - grp.affinity().calculate(fut.topologyVersion(), fut.discoveryEvent(), fut.discoCache()); + cache.aff.calculate(fut.topologyVersion(), fut.discoveryEvent(), fut.discoCache()); + } + }); + } + else { + forAllCacheGroups(false, new IgniteInClosureX() { + @Override public void applyx(GridAffinityAssignmentCache aff) throws IgniteCheckedException { + aff.calculate(fut.topologyVersion(), fut.discoveryEvent(), fut.discoCache()); + } + }); } synchronized (mux) { @@ -1433,12 +1468,8 @@ private IgniteInternalFuture initCoordinatorCaches(final GridDhtPartitionsExc @Override public void applyx(CacheGroupDescriptor desc) throws IgniteCheckedException { CacheGroupHolder grpHolder = grpHolders.get(desc.groupId()); - if (grpHolder != null) { - if (grpHolder.client()) // Affinity for non-client holders calculated in {@link #onServerLeft}. - grpHolder.affinity().calculate(fut.topologyVersion(), fut.discoveryEvent(), fut.discoCache()); - + if (grpHolder != null) return; - } // Need initialize holders and affinity if this node became coordinator during this exchange. final Integer grpId = desc.groupId(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java index f4ed517975155..232ce383e59f5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java @@ -366,6 +366,11 @@ else if (!node2part.nodeId().equals(loc.id())) { return localPartition(p, topVer, create); } + /** {@inheritDoc} */ + @Override public GridDhtLocalPartition forceCreatePartition(int p) throws IgniteCheckedException { + throw new UnsupportedOperationException(); + } + /** {@inheritDoc} */ @Override public GridDhtLocalPartition localPartition(int p) { return localPartition(p, AffinityTopologyVersion.NONE, false); @@ -830,7 +835,7 @@ private boolean isStaleUpdate(GridDhtPartitionMap currentMap, GridDhtPartitionMa } /** {@inheritDoc} */ - @Override public void onExchangeDone(AffinityAssignment assignment) { + @Override public void onExchangeDone(AffinityAssignment assignment, boolean updateRebalanceVer) { // No-op. } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java index 5f76d12a16fae..d9e04a6da4a46 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java @@ -129,6 +129,15 @@ public void beforeExchange(GridDhtPartitionsExchangeFuture exchFut, boolean affR @Nullable public GridDhtLocalPartition localPartition(int p, AffinityTopologyVersion topVer, boolean create) throws GridDhtInvalidPartitionException; + /** + * Unconditionally creates partition during restore of persisted partition state. + * + * @param p Partition ID. + * @return Partition. + * @throws IgniteCheckedException If failed. + */ + public GridDhtLocalPartition forceCreatePartition(int p) throws IgniteCheckedException; + /** * @param topVer Topology version at the time of creation. * @param p Partition ID. @@ -331,6 +340,7 @@ public boolean update(@Nullable GridDhtPartitionExchangeId exchId, * Callback on exchange done. * * @param assignment New affinity assignment. + * @param updateRebalanceVer {@code True} if need check rebalance state. */ - public void onExchangeDone(AffinityAssignment assignment); + public void onExchangeDone(AffinityAssignment assignment, boolean updateRebalanceVer); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java index 601da1b4ea2f4..5ef499cf32516 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java @@ -135,9 +135,6 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology { /** */ private volatile AffinityTopologyVersion rebalancedTopVer = AffinityTopologyVersion.NONE; - /** */ - private volatile boolean treatAllPartAsLoc; - /** * @param ctx Cache shared context. * @param grp Cache group. @@ -421,14 +418,6 @@ else if (localNode(p, aff)) /** {@inheritDoc} */ @Override public void beforeExchange(GridDhtPartitionsExchangeFuture exchFut, boolean affReady) throws IgniteCheckedException { - DiscoveryEvent discoEvt = exchFut.discoveryEvent(); - - treatAllPartAsLoc = exchFut.activateCluster() - || (discoEvt.type() == EventType.EVT_NODE_JOINED - && discoEvt.eventNode().isLocal() - && !ctx.kernalContext().clientNode() - ); - ClusterNode loc = ctx.localNode(); ctx.database().checkpointReadLock(); @@ -540,8 +529,6 @@ private boolean partitionLocalNode(int p, AffinityTopologyVersion topVer) { /** {@inheritDoc} */ @Override public boolean afterExchange(GridDhtPartitionsExchangeFuture exchFut) throws IgniteCheckedException { - treatAllPartAsLoc = false; - boolean changed = false; int num = grp.affinity().partitions(); @@ -692,6 +679,29 @@ private GridDhtLocalPartition createPartition(int p) { return loc; } + /** {@inheritDoc} */ + @Override public GridDhtLocalPartition forceCreatePartition(int p) throws IgniteCheckedException { + lock.writeLock().lock(); + + try { + GridDhtLocalPartition part = locParts.get(p); + + if (part != null) + return part; + + part = new GridDhtLocalPartition(ctx, grp, p); + + locParts.set(p, part); + + ctx.pageStore().onPartitionCreated(grp.groupId(), p); + + return part; + } + finally { + lock.writeLock().unlock(); + } + } + /** * @param p Partition number. * @param topVer Topology version. @@ -731,7 +741,7 @@ private GridDhtLocalPartition localPartition0(int p, if (loc != null && state == EVICTED) { locParts.set(p, loc = null); - if (!treatAllPartAsLoc && !belongs) + if (!belongs) throw new GridDhtInvalidPartitionException(p, "Adding entry to evicted partition " + "(often may be caused by inconsistent 'key.hashCode()' implementation) " + "[part=" + p + ", topVer=" + topVer + ", this.topVer=" + this.topVer + ']'); @@ -741,7 +751,7 @@ else if (loc != null && state == RENTING && !showRenting) "[part=" + p + ", shouldBeMoving=" + loc.reload() + "]"); if (loc == null) { - if (!treatAllPartAsLoc && !belongs) + if (!belongs) throw new GridDhtInvalidPartitionException(p, "Creating partition which does not belong to " + "local node (often may be caused by inconsistent 'key.hashCode()' implementation) " + "[part=" + p + ", topVer=" + topVer + ", this.topVer=" + this.topVer + ']'); @@ -1499,12 +1509,15 @@ private boolean isStaleUpdate(GridDhtPartitionMap currentMap, GridDhtPartitionMa } /** {@inheritDoc} */ - @Override public void onExchangeDone(AffinityAssignment assignment) { + @Override public void onExchangeDone(AffinityAssignment assignment, boolean updateRebalanceVer) { lock.writeLock().lock(); try { if (assignment.topologyVersion().compareTo(diffFromAffinityVer) >= 0) rebuildDiff(assignment); + + if (updateRebalanceVer) + updateRebalanceVersion(assignment.assignment()); } finally { lock.writeLock().unlock(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index c4a4f833c09cd..cdb4bb7086095 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -193,9 +193,6 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte /** */ private CacheAffinityChangeMessage affChangeMsg; - /** */ - private boolean clientOnlyExchange; - /** Init timestamp. Used to track the amount of time spent to complete the future. */ private long initTs; @@ -485,26 +482,8 @@ else if (msg instanceof SnapshotDiscoveryMessage) { cctx.affinity().initStartedCaches(crdNode, this, receivedCaches); } - else { - cctx.activate(); - - List> caches = - cctx.cache().cachesToStartOnLocalJoin(); - - if (cctx.database().persistenceEnabled() && - !cctx.kernalContext().clientNode()) { - List startDescs = new ArrayList<>(); - - if (caches != null) { - for (T2 c : caches) - startDescs.add(c.get1()); - } - - cctx.database().readCheckpointAndRestoreMemory(startDescs); - } - - cctx.cache().startCachesOnLocalJoin(caches, topVer); - } + else + initCachesOnLocalJoin(); } exchange = CU.clientNode(discoEvt.eventNode()) ? @@ -568,6 +547,29 @@ else if (msg instanceof SnapshotDiscoveryMessage) { } } + /** + * @throws IgniteCheckedException If failed. + */ + private void initCachesOnLocalJoin() throws IgniteCheckedException { + cctx.activate(); + + List> caches = + cctx.cache().cachesToStartOnLocalJoin(); + + if (cctx.database().persistenceEnabled() && !cctx.kernalContext().clientNode()) { + List startDescs = new ArrayList<>(); + + if (caches != null) { + for (T2 c : caches) + startDescs.add(c.get1()); + } + + cctx.database().readCheckpointAndRestoreMemory(startDescs); + } + + cctx.cache().startCachesOnLocalJoin(caches, topologyVersion()); + } + /** * @throws IgniteCheckedException If failed. */ @@ -776,7 +778,7 @@ private ExchangeType onServerNodeEvent(boolean crd) throws IgniteCheckedExceptio warnNoAffinityNodes(); - centralizedAff = cctx.affinity().onServerLeft(this); + centralizedAff = cctx.affinity().onServerLeft(this, crd); } else cctx.affinity().onServerJoin(this, crd); @@ -788,40 +790,15 @@ private ExchangeType onServerNodeEvent(boolean crd) throws IgniteCheckedExceptio * @throws IgniteCheckedException If failed. */ private void clientOnlyExchange() throws IgniteCheckedException { - clientOnlyExchange = true; - if (crd != null) { - if (crd.isLocal()) { - for (CacheGroupContext grp : cctx.cache().cacheGroups()) { - boolean updateTop = !grp.isLocal() && - exchId.topologyVersion().equals(grp.localStartVersion()); - - if (updateTop) { - for (GridClientPartitionTopology top : cctx.exchange().clientTopologies()) { - if (top.groupId() == grp.groupId()) { - GridDhtPartitionFullMap fullMap = top.partitionMap(true); - - assert fullMap != null; - - grp.topology().update(topologyVersion(), - fullMap, - top.updateCounters(false), - Collections.emptySet()); + assert !crd.isLocal() : crd; - break; - } - } - } - } - } - else { - if (!centralizedAff) - sendLocalPartitions(crd); + if (!centralizedAff) + sendLocalPartitions(crd); - initDone(); + initDone(); - return; - } + return; } else { if (centralizedAff) { // Last server node failed. @@ -896,8 +873,7 @@ private void tryToPerformLocalSnapshotOperation() { try { long start = U.currentTimeMillis(); - IgniteInternalFuture fut = cctx.snapshot() - .tryStartLocalSnapshotOperation(discoEvt); + IgniteInternalFuture fut = cctx.snapshot().tryStartLocalSnapshotOperation(discoEvt); if (fut != null) { fut.get(); @@ -1122,6 +1098,8 @@ private boolean cacheStopping(int cacheId) { private void sendLocalPartitions(ClusterNode node) throws IgniteCheckedException { assert node != null; + GridDhtPartitionsSingleMessage msg; + // Reset lost partition before send local partition to coordinator. if (exchActions != null) { Set caches = exchActions.cachesToResetLostPartitions(); @@ -1130,22 +1108,32 @@ private void sendLocalPartitions(ClusterNode node) throws IgniteCheckedException resetLostPartitions(caches); } - GridDhtPartitionsSingleMessage m = cctx.exchange().createPartitionsSingleMessage( - node, exchangeId(), clientOnlyExchange, true); + if (cctx.kernalContext().clientNode()) { + msg = new GridDhtPartitionsSingleMessage(exchangeId(), + true, + null, + true); + } + else { + msg = cctx.exchange().createPartitionsSingleMessage(node, + exchangeId(), + false, + true); + } Map> partHistReserved0 = partHistReserved; if (partHistReserved0 != null) - m.partitionHistoryCounters(partHistReserved0); + msg.partitionHistoryCounters(partHistReserved0); if (stateChangeExchange() && changeGlobalStateE != null) - m.setError(changeGlobalStateE); + msg.setError(changeGlobalStateE); if (log.isDebugEnabled()) - log.debug("Sending local partitions [nodeId=" + node.id() + ", exchId=" + exchId + ", msg=" + m + ']'); + log.debug("Sending local partitions [nodeId=" + node.id() + ", exchId=" + exchId + ", msg=" + msg + ']'); try { - cctx.io().send(node, m, SYSTEM_POOL); + cctx.io().send(node, msg, SYSTEM_POOL); } catch (ClusterTopologyCheckedException ignored) { if (log.isDebugEnabled()) @@ -1318,7 +1306,7 @@ public boolean serverNodeDiscoveryEvent() { if (err == null) { for (CacheGroupContext grp : cctx.cache().cacheGroups()) { if (!grp.isLocal()) - grp.topology().onExchangeDone(grp.affinity().cachedAffinity(topologyVersion())); + grp.topology().onExchangeDone(grp.affinity().cachedAffinity(topologyVersion()), false); } } @@ -1386,10 +1374,12 @@ private void updateLastVersion(GridCacheVersion ver) { public void onReceive(final ClusterNode node, final GridDhtPartitionsSingleMessage msg) { assert msg != null; assert msg.exchangeId().equals(exchId) : msg; - assert msg.lastVersion() != null : msg; - if (!msg.client()) + if (!msg.client()) { + assert msg.lastVersion() != null : msg; + updateLastVersion(msg.lastVersion()); + } if (isDone()) { if (log.isDebugEnabled()) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 39038babdd9e3..1797d641b0738 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -1560,8 +1560,7 @@ private void restorePartitionState( T2 fromWal = partStates.get(new T2<>(grpId, i)); - GridDhtLocalPartition part = grp.topology() - .localPartition(i, AffinityTopologyVersion.NONE, true); + GridDhtLocalPartition part = grp.topology().forceCreatePartition(i); assert part != null; @@ -1621,8 +1620,7 @@ private boolean updateState(GridDhtLocalPartition part, int stateId) { * @param dataEntry Data entry to apply. */ private void applyUpdate(GridCacheContext cacheCtx, DataEntry dataEntry) throws IgniteCheckedException { - GridDhtLocalPartition locPart = cacheCtx.topology() - .localPartition(dataEntry.partitionId(), AffinityTopologyVersion.NONE, true); + GridDhtLocalPartition locPart = cacheCtx.topology().forceCreatePartition(dataEntry.partitionId()); switch (dataEntry.op()) { case CREATE: diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java index 23043d10f773f..7d8620a210502 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java @@ -331,7 +331,7 @@ public void testAffinitySimpleSequentialStartNoCacheOnCoordinator() throws Excep } }; - cacheNodeFilter = new CacheNodeFilter(F.asList(getTestIgniteInstanceName(0))); + cacheNodeFilter = new TestCacheNodeExcludingFilter(F.asList(getTestIgniteInstanceName(0))); testAffinitySimpleSequentialStart(); @@ -351,7 +351,7 @@ public void testAffinitySimpleNoCacheOnCoordinator1() throws Exception { } }; - cacheNodeFilter = new CacheNodeFilter(F.asList(getTestIgniteInstanceName(1))); + cacheNodeFilter = new TestCacheNodeExcludingFilter(F.asList(getTestIgniteInstanceName(1))); startServer(0, 1); @@ -391,7 +391,7 @@ public void testAffinitySimpleNoCacheOnCoordinator2() throws Exception { } }; - cacheNodeFilter = new CacheNodeFilter(F.asList(getTestIgniteInstanceName(1), getTestIgniteInstanceName(2))); + cacheNodeFilter = new TestCacheNodeExcludingFilter(F.asList(getTestIgniteInstanceName(1), getTestIgniteInstanceName(2))); startServer(0, 1); startServer(1, 2); @@ -439,7 +439,7 @@ public void testCreateCloseClientCacheOnCoordinator1() throws Exception { } }; - cacheNodeFilter = new CacheNodeFilter(F.asList(getTestIgniteInstanceName(0))); + cacheNodeFilter = new TestCacheNodeExcludingFilter(F.asList(getTestIgniteInstanceName(0))); Ignite ignite0 = startServer(0, 1); @@ -467,7 +467,7 @@ public void testCreateCloseClientCacheOnCoordinator2() throws Exception { } }; - cacheNodeFilter = new CacheNodeFilter(F.asList(getTestIgniteInstanceName(0))); + cacheNodeFilter = new TestCacheNodeExcludingFilter(F.asList(getTestIgniteInstanceName(0))); Ignite ignite0 = startServer(0, 1); @@ -520,7 +520,7 @@ public void testCacheDestroyAndCreate2() throws Exception { */ private void cacheDestroyAndCreate(boolean cacheOnCrd) throws Exception { if (!cacheOnCrd) - cacheNodeFilter = new CacheNodeFilter(Collections.singletonList(getTestIgniteInstanceName(0))); + cacheNodeFilter = new TestCacheNodeExcludingFilter(Collections.singletonList(getTestIgniteInstanceName(0))); startServer(0, 1); @@ -2069,7 +2069,7 @@ private CacheConfiguration randomCacheConfiguration(Random rnd, String name, Lis exclude.add("server-" + (srvIdx + rnd.nextInt(10))); } - ccfg.setNodeFilter(new CacheNodeFilter(exclude)); + ccfg.setNodeFilter(new TestCacheNodeExcludingFilter(exclude)); } ccfg.setName(name); @@ -2642,28 +2642,6 @@ public TestServiceImpl(int key) { } } - /** - * - */ - static class CacheNodeFilter implements IgnitePredicate { - /** */ - private Collection excludeNodes; - - /** - * @param excludeNodes Nodes names. - */ - public CacheNodeFilter(Collection excludeNodes) { - this.excludeNodes = excludeNodes; - } - - /** {@inheritDoc} */ - @Override public boolean apply(ClusterNode clusterNode) { - String name = clusterNode.attribute(ATTR_IGNITE_INSTANCE_NAME).toString(); - - return !excludeNodes.contains(name); - } - } - /** * */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionStateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionStateTest.java new file mode 100644 index 0000000000000..c64ed0b6b533a --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionStateTest.java @@ -0,0 +1,410 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed; + +import java.util.HashSet; +import java.util.Set; +import org.apache.ignite.Ignite; +import org.apache.ignite.cache.affinity.Affinity; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteKernal; +import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.processors.affinity.AffinityAssignment; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.processors.cache.GridCacheAdapter; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.lang.IgniteBiPredicate; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; +import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.EVICTED; +import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.MOVING; +import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING; + +/** + * + */ +public class CachePartitionStateTest extends GridCommonAbstractTest { + /** */ + private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); + + /** */ + private boolean client; + + /** */ + private CacheConfiguration ccfg; + + /** {@inheritDoc} */ + protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder); + + cfg.setCommunicationSpi(new TestRecordingCommunicationSpi()); + + cfg.setClientMode(client); + + if (ccfg != null) { + cfg.setCacheConfiguration(ccfg); + + ccfg = null; + } + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + super.afterTest(); + } + + /** + * @throws Exception If failed. + */ + public void testPartitionState1_1() throws Exception { + partitionState1(0, true); + } + + /** + * @throws Exception If failed. + */ + public void testPartitionState1_2() throws Exception { + partitionState1(1, true); + } + + /** + * @throws Exception If failed. + */ + public void testPartitionState1_2_NoCacheOnCoordinator() throws Exception { + partitionState1(1, false); + } + + /** + * @throws Exception If failed. + */ + public void testPartitionState1_3() throws Exception { + partitionState1(100, true); + } + + /** + * @throws Exception If failed. + */ + public void testPartitionState2_1() throws Exception { + partitionState2(0, true); + } + + /** + * @throws Exception If failed. + */ + public void testPartitionState2_2() throws Exception { + partitionState2(1, true); + } + + /** + * @throws Exception If failed. + */ + public void testPartitionState2_2_NoCacheOnCoordinator() throws Exception { + partitionState2(1, false); + } + + /** + * @throws Exception If failed. + */ + public void testPartitionState2_3() throws Exception { + partitionState2(100, true); + } + + /** + * @param backups Number of backups. + * @param crdAffNode If {@code false} cache is not created on coordinator. + * @throws Exception If failed. + */ + private void partitionState1(int backups, boolean crdAffNode) throws Exception { + startGrids(3); + + blockSupplySend(DEFAULT_CACHE_NAME); + + CacheConfiguration ccfg = cacheConfiguration(DEFAULT_CACHE_NAME, backups); + + if (!crdAffNode) + ccfg.setNodeFilter(new TestCacheNodeExcludingFilter(getTestIgniteInstanceName(0))); + + ignite(1).createCache(ccfg); + + AffinityAssignment assign0 = + grid(1).context().cache().internalCache(DEFAULT_CACHE_NAME).context().affinity().assignment( + new AffinityTopologyVersion(3, 1)); + + awaitPartitionMapExchange(); + + checkPartitionsState(assign0, DEFAULT_CACHE_NAME, OWNING); + + checkRebalance(DEFAULT_CACHE_NAME, true); + + client = true; + + Ignite clientNode = startGrid(4); + + checkPartitionsState(assign0, DEFAULT_CACHE_NAME, OWNING); + + clientNode.cache(DEFAULT_CACHE_NAME); + + checkPartitionsState(assign0, DEFAULT_CACHE_NAME, OWNING); + + checkRebalance(DEFAULT_CACHE_NAME, true); + + client = false; + + startGrid(5); + + checkRebalance(DEFAULT_CACHE_NAME, false); + + for (int i = 0; i < 3; i++) + checkNodePartitions(assign0, ignite(i).cluster().localNode(), DEFAULT_CACHE_NAME, OWNING); + + AffinityAssignment assign1 = + grid(1).context().cache().internalCache(DEFAULT_CACHE_NAME).context().affinity().assignment( + new AffinityTopologyVersion(5, 0)); + + checkNodePartitions(assign1, ignite(5).cluster().localNode(), DEFAULT_CACHE_NAME, MOVING); + + stopBlock(); + + awaitPartitionMapExchange(); + + AffinityAssignment assign2 = + grid(1).context().cache().internalCache(DEFAULT_CACHE_NAME).context().affinity().assignment( + new AffinityTopologyVersion(5, 1)); + + checkPartitionsState(assign2, DEFAULT_CACHE_NAME, OWNING); + + checkRebalance(DEFAULT_CACHE_NAME, true); + + if (!crdAffNode) + ignite(0).cache(DEFAULT_CACHE_NAME); + + checkPartitionsState(assign2, DEFAULT_CACHE_NAME, OWNING); + + checkRebalance(DEFAULT_CACHE_NAME, true); + + startGrid(6); + + awaitPartitionMapExchange(); + + AffinityAssignment assign3 = + grid(1).context().cache().internalCache(DEFAULT_CACHE_NAME).context().affinity().assignment( + new AffinityTopologyVersion(6, 1)); + + checkPartitionsState(assign3, DEFAULT_CACHE_NAME, OWNING); + + checkRebalance(DEFAULT_CACHE_NAME, true); + } + + /** + * @param backups Number of backups. + * @param crdAffNode If {@code false} cache is not created on coordinator. + * @throws Exception If failed. + */ + private void partitionState2(int backups, boolean crdAffNode) throws Exception { + startGrids(3); + + blockSupplySend(DEFAULT_CACHE_NAME); + + ccfg = cacheConfiguration(DEFAULT_CACHE_NAME, backups); + + if (!crdAffNode) + ccfg.setNodeFilter(new TestCacheNodeExcludingFilter(getTestIgniteInstanceName(0))); + + startGrid(4); + + AffinityAssignment assign0 = + grid(1).context().cache().internalCache(DEFAULT_CACHE_NAME).context().affinity().assignment( + new AffinityTopologyVersion(4, 0)); + + checkPartitionsState(assign0, DEFAULT_CACHE_NAME, OWNING); + + checkRebalance(DEFAULT_CACHE_NAME, true); + + if (!crdAffNode) + ignite(0).cache(DEFAULT_CACHE_NAME); + + checkPartitionsState(assign0, DEFAULT_CACHE_NAME, OWNING); + + checkRebalance(DEFAULT_CACHE_NAME, true); + + stopBlock(); + + startGrid(5); + + AffinityAssignment assign1 = + grid(1).context().cache().internalCache(DEFAULT_CACHE_NAME).context().affinity().assignment( + new AffinityTopologyVersion(5, 1)); + + awaitPartitionMapExchange(); + + checkPartitionsState(assign1, DEFAULT_CACHE_NAME, OWNING); + + checkRebalance(DEFAULT_CACHE_NAME, true); + } + + /** + * @param assign Assignments. + * @param cacheName Cache name. + * @param expState Expected state. + */ + private void checkPartitionsState(AffinityAssignment assign, String cacheName, GridDhtPartitionState expState) { + for (Ignite node : G.allGrids()) + checkNodePartitions(assign, node.cluster().localNode(), cacheName, expState); + } + + /** + * @param assign Assignments. + * @param clusterNode Node. + * @param cacheName Cache name. + * @param expState Expected partitions state. + */ + private void checkNodePartitions(AffinityAssignment assign, + ClusterNode clusterNode, + String cacheName, + GridDhtPartitionState expState) + { + Affinity aff = ignite(0).affinity(cacheName); + + Set nodeParts = new HashSet<>(); + + nodeParts.addAll(assign.primaryPartitions(clusterNode.id())); + nodeParts.addAll(assign.backupPartitions(clusterNode.id())); + + log.info("Test state [node=" + clusterNode.id() + ", parts=" + nodeParts.size() + ", state=" + expState + ']'); + + if (grid(0).context().discovery().cacheAffinityNode(clusterNode, cacheName)) + assertFalse(nodeParts.isEmpty()); + + boolean check = false; + + for (Ignite node : G.allGrids()) { + GridCacheAdapter cache = + ((IgniteKernal)node).context().cache().internalCache(cacheName); + + if (cache != null) { + check = true; + + GridDhtPartitionTopology top = cache.context().topology(); + + GridDhtPartitionMap partsMap = top.partitions(clusterNode.id()); + + for (int p = 0; p < aff.partitions(); p++) { + if (nodeParts.contains(p)) { + assertNotNull(partsMap); + assertEquals(expState, partsMap.get(p)); + } + else { + if (partsMap != null) { + GridDhtPartitionState state = partsMap.get(p); + + assertTrue("Unexpected state: " + state, state == null || state == EVICTED); + } + } + } + } + else { + assertEquals(0, aff.primaryPartitions(((IgniteKernal)node).localNode()).length); + assertEquals(0, aff.backupPartitions(((IgniteKernal)node).localNode()).length); + } + } + + assertTrue(check); + } + + /** + * @param cacheName Cache name. + * @param expDone Expected rebalance finish flag. + */ + private void checkRebalance(String cacheName, boolean expDone) { + for (Ignite node : G.allGrids()) { + IgniteKernal node0 = (IgniteKernal)node; + + GridCacheAdapter cache = node0.context().cache().internalCache(cacheName); + + AffinityTopologyVersion topVer = node0.context().cache().context().exchange().readyAffinityVersion(); + + if (cache != null) + assertEquals(expDone, cache.context().topology().rebalanceFinished(topVer)); + else + node0.context().discovery().cacheAffinityNode(node0.localNode(), cacheName); + } + } + + /** + * @param cacheName Cache name. + */ + private void blockSupplySend(String cacheName) { + for (Ignite node : G.allGrids()) + blockSupplySend(TestRecordingCommunicationSpi.spi(node), cacheName); + } + + /** + * @param spi SPI. + * @param cacheName Cache name. + */ + private void blockSupplySend(TestRecordingCommunicationSpi spi, final String cacheName) { + final int grpId = CU.cacheId(cacheName); + + spi.blockMessages(new IgniteBiPredicate() { + @Override public boolean apply(ClusterNode node, Message msg) { + return msg.getClass().equals(GridDhtPartitionSupplyMessage.class) && + ((GridDhtPartitionSupplyMessage)msg).groupId() == grpId; + } + }); + } + + /** + * + */ + private void stopBlock() { + for (Ignite node : G.allGrids()) + TestRecordingCommunicationSpi.spi(node).stopBlock(); + } + + /** + * @param name Cache name. + * @param backups Backups number. + * @return Cache configuration. + */ + private CacheConfiguration cacheConfiguration(String name, int backups) { + CacheConfiguration ccfg = new CacheConfiguration(name); + + ccfg.setWriteSynchronizationMode(FULL_SYNC); + ccfg.setBackups(backups); + + return ccfg; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/TestCacheNodeExcludingFilter.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/TestCacheNodeExcludingFilter.java new file mode 100644 index 0000000000000..a3f7d270ffe98 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/TestCacheNodeExcludingFilter.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed; + +import java.util.Arrays; +import java.util.Collection; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.lang.IgnitePredicate; + +import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGNITE_INSTANCE_NAME; + +/** + * + */ +public class TestCacheNodeExcludingFilter implements IgnitePredicate { + /** */ + private Collection excludeNodes; + + /** + * @param excludeNodes Nodes names. + */ + public TestCacheNodeExcludingFilter(Collection excludeNodes) { + this.excludeNodes = excludeNodes; + } + /** + * @param excludeNodes Nodes names. + */ + public TestCacheNodeExcludingFilter(String... excludeNodes) { + this.excludeNodes = Arrays.asList(excludeNodes); + } + + /** {@inheritDoc} */ + @Override public boolean apply(ClusterNode clusterNode) { + String name = clusterNode.attribute(ATTR_IGNITE_INSTANCE_NAME).toString(); + + return !excludeNodes.contains(name); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheRestoreTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheRestoreTest.java new file mode 100644 index 0000000000000..25626f4bb33c3 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheRestoreTest.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.db; + +import java.util.Arrays; +import java.util.List; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.MemoryConfiguration; +import org.apache.ignite.configuration.PersistentStoreConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; + +/** + * + */ +public class IgnitePdsCacheRestoreTest extends GridCommonAbstractTest { + /** */ + private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** */ + private CacheConfiguration[] ccfgs; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER); + + if (ccfgs != null) { + cfg.setCacheConfiguration(ccfgs); + + ccfgs = null; + } + + MemoryConfiguration memCfg = new MemoryConfiguration(); + memCfg.setPageSize(1024); + memCfg.setDefaultMemoryPolicySize(10 * 1024 * 1024); + + cfg.setMemoryConfiguration(memCfg); + + PersistentStoreConfiguration pCfg = new PersistentStoreConfiguration(); + + pCfg.setWalMode(WALMode.LOG_ONLY); + + cfg.setPersistentStoreConfiguration(pCfg); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + GridTestUtils.deleteDbFiles(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + GridTestUtils.deleteDbFiles(); + + super.afterTest(); + } + + /** + * @throws Exception If failed. + */ + public void testRestoreAndNewCache1() throws Exception { + restoreAndNewCache(false); + } + + /** + * @throws Exception If failed. + */ + public void testRestoreAndNewCache2() throws Exception { + restoreAndNewCache(true); + } + + /** + * @param createNew If {@code true} need cache is added while node is stopped. + * @throws Exception If failed. + */ + private void restoreAndNewCache(boolean createNew) throws Exception { + for (int i = 0; i < 3; i++) { + ccfgs = configurations1(); + + startGrid(i); + } + + ignite(0).active(true); + + IgniteCache cache1 = ignite(2).cache("c1"); + + List keys = primaryKeys(cache1, 10); + + for (Integer key : keys) + cache1.put(key, key); + + stopGrid(2); + + if (createNew) { + // New cache is added when node is stopped. + ignite(0).getOrCreateCaches(Arrays.asList(configurations2())); + } + else { + // New cache is added on node restart. + ccfgs = configurations2(); + } + + startGrid(2); + + cache1 = ignite(2).cache("c1"); + + IgniteCache cache2 = ignite(2).cache("c2"); + + for (Integer key : keys) { + assertEquals(key, cache1.get(key)); + + assertNull(cache2.get(key)); + + cache2.put(key, key); + + assertEquals(key, cache2.get(key)); + } + + List nearKeys = nearKeys(cache1, 10, 0); + + for (Integer key : nearKeys) { + assertNull(cache1.get(key)); + assertNull(cache2.get(key)); + + cache2.put(key, key); + assertEquals(key, cache2.get(key)); + + cache1.put(key, key); + assertEquals(key, cache1.get(key)); + } + + startGrid(3); + + awaitPartitionMapExchange(); + + for (Integer key : nearKeys) { + assertEquals(key, cache2.get(key)); + + assertEquals(key, cache1.get(key)); + } + } + + /** + * @return Configurations set 1. + */ + private CacheConfiguration[] configurations1() { + CacheConfiguration[] ccfgs = new CacheConfiguration[1]; + + ccfgs[0] = cacheConfiguration("c1"); + + return ccfgs; + } + + /** + * @return Configurations set 1. + */ + private CacheConfiguration[] configurations2() { + CacheConfiguration[] ccfgs = new CacheConfiguration[2]; + + ccfgs[0] = cacheConfiguration("c1"); + ccfgs[1] = cacheConfiguration("c2"); + + return ccfgs; + } + + /** + * @param name Cache name. + * @return Cache configuration. + */ + private CacheConfiguration cacheConfiguration(String name) { + CacheConfiguration ccfg = new CacheConfiguration(name); + + ccfg.setWriteSynchronizationMode(FULL_SYNC); + + return ccfg; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java new file mode 100644 index 0000000000000..bb32d24a65457 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testsuites; + +import junit.framework.TestSuite; +import org.apache.ignite.internal.processors.cache.distributed.CachePartitionStateTest; + +/** + * Test suite. + */ +public class IgniteCacheTestSuite6 extends TestSuite { + /** + * @return IgniteCache test suite. + * @throws Exception Thrown in case of the failure. + */ + public static TestSuite suite() throws Exception { + TestSuite suite = new TestSuite("IgniteCache Test Suite part 6"); + + suite.addTestSuite(CachePartitionStateTest.class); + + return suite; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java index 5b562c3eabdf5..5762c0299376f 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java @@ -22,6 +22,7 @@ import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsClientNearCachePutGetTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDynamicCacheTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsSingleNodePutGetPersistenceTest; +import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsCacheRestoreTest; import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsCheckpointSimulationWithRealCpDisabledTest; import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsEvictionTest; import org.apache.ignite.internal.processors.cache.persistence.pagemem.BPlusTreePageMemoryImplTest; @@ -74,6 +75,8 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(IgniteClusterActivateDeactivateTestWithPersistence.class); + suite.addTestSuite(IgnitePdsCacheRestoreTest.class); + return suite; } } From 24e4d5d31cc494c8c55000aafefd119cc9d879a4 Mon Sep 17 00:00:00 2001 From: Evgeniy Ignatiev Date: Mon, 24 Jul 2017 15:45:10 +0300 Subject: [PATCH 017/547] IGNITE-5123 Reorder ignite plugin provider on Ignite start callback call --- .../java/org/apache/ignite/internal/IgniteKernal.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java index 00c1d73773cf3..51864098569f6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java @@ -1015,10 +1015,6 @@ public void start( // Notify IO manager the second so further components can send and receive messages. ctx.io().onKernalStart(active); - // Start plugins. - for (PluginProvider provider : ctx.plugins().allProviders()) - provider.onIgniteStart(); - boolean recon = false; // Callbacks. @@ -1049,6 +1045,10 @@ public void start( } } + // Start plugins. + for (PluginProvider provider : ctx.plugins().allProviders()) + provider.onIgniteStart(); + if (recon) reconnectState.waitFirstReconnect(); From 301943361bd29c9c33b7ba15e9567a0e05bf27e2 Mon Sep 17 00:00:00 2001 From: sboikov Date: Mon, 24 Jul 2017 23:46:57 +0300 Subject: [PATCH 018/547] ignite-5805 Fixed some lgtm.com analysis alerts --- .../configuration/CacheConfiguration.java | 5 -- .../internal/MarshallerContextImpl.java | 4 +- .../affinity/GridAffinityProcessor.java | 2 +- .../cache/CacheAffinitySharedManager.java | 8 +-- .../CacheObjectBinaryProcessorImpl.java | 5 -- .../dht/GridClientPartitionTopology.java | 11 ++-- .../distributed/dht/GridDhtGetFuture.java | 14 ++-- .../dht/atomic/GridDhtAtomicCache.java | 2 +- .../cache/local/GridLocalLockFuture.java | 5 -- .../freelist/io/PagesListMetaIO.java | 2 +- .../cache/transactions/IgniteTxHandler.java | 66 +++++++++---------- .../cache/transactions/IgniteTxManager.java | 2 +- .../processors/igfs/IgfsDataManager.java | 2 +- .../GridRedisIncrDecrCommandHandler.java | 2 +- .../ignite/internal/util/IgniteUtils.java | 16 ++--- .../internal/util/nio/GridNioServer.java | 14 ++-- .../cache/VisorCacheMetricsCollectorTask.java | 2 +- .../tcp/TcpCommunicationSpi.java | 25 +++++-- .../ignite/spi/discovery/tcp/ServerImpl.java | 2 +- .../failover/always/AlwaysFailoverSpi.java | 2 +- .../ignite/mesos/ClusterProperties.java | 4 +- .../cache/websession/WebSessionFilter.java | 2 +- 22 files changed, 100 insertions(+), 97 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java index 670046f508436..708913ab6da08 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java @@ -2075,11 +2075,6 @@ private static QueryEntity convert(TypeDescriptor desc) { txtIdx.setName(idxEntry.getKey()); } else { - Collection grp = new ArrayList<>(); - - for (String fieldName : idx.fields()) - grp.add(idx.descending(fieldName) ? fieldName + " desc" : fieldName); - QueryIndex sortedIdx = new QueryIndex(); sortedIdx.setIndexType(idx.type()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java index 6f1550792fd0d..bb93354971899 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java @@ -197,9 +197,7 @@ public void checkHasClassName(String clsName, ClassLoader ldr, String fileName) * @throws IOException In case of error. */ private void processResource(URL url) throws IOException { - try (InputStream in = url.openStream()) { - BufferedReader rdr = new BufferedReader(new InputStreamReader(in)); - + try (BufferedReader rdr = new BufferedReader(new InputStreamReader(url.openStream()))) { String line; while ((line = rdr.readLine()) != null) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java index b2989d3db5850..4ee050291aaaa 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java @@ -471,7 +471,7 @@ private Map> keysToNodes(@Nullable final String c continue; } - affMap.remove(cacheName, fut0); + affMap.remove(key, fut0); fut0.onDone(new IgniteCheckedException("Failed to get affinity mapping from node: " + n, e)); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java index f519b4e8fbff6..0f46a906cfbf6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java @@ -762,7 +762,7 @@ public void onCacheChangeRequest( Set gprs = new HashSet<>(); for (ExchangeActions.CacheActionData action : exchActions.cacheStartRequests()) { - Integer grpId = action.descriptor().groupId(); + int grpId = action.descriptor().groupId(); if (gprs.add(grpId)) { if (crd) @@ -1109,7 +1109,7 @@ private void initStartedGroupOnCoordinator(GridDhtPartitionsExchangeFuture fut, if (grpDesc.config().getCacheMode() == LOCAL) return; - Integer grpId = grpDesc.groupId(); + int grpId = grpDesc.groupId(); CacheGroupHolder grpHolder = grpHolders.get(grpId); @@ -1360,7 +1360,7 @@ private void fetchAffinityOnJoin(GridDhtPartitionsExchangeFuture fut) throws Ign for (int i = 0; i < fetchFuts.size(); i++) { GridDhtAssignmentFetchFuture fetchFut = fetchFuts.get(i); - Integer grpId = fetchFut.groupId(); + int grpId = fetchFut.groupId(); fetchAffinity(fut.topologyVersion(), fut.discoveryEvent(), @@ -1472,7 +1472,7 @@ private IgniteInternalFuture initCoordinatorCaches(final GridDhtPartitionsExc return; // Need initialize holders and affinity if this node became coordinator during this exchange. - final Integer grpId = desc.groupId(); + int grpId = desc.groupId(); CacheGroupContext grp = cctx.cache().cacheGroup(grpId); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java index c0f3515c47e0d..6a1b6dc7a5e54 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java @@ -541,11 +541,6 @@ public GridBinaryMarshaller marshaller() { @Override public Map metadata(Collection typeIds) throws BinaryObjectException { try { - Collection keys = new ArrayList<>(typeIds.size()); - - for (Integer typeId : typeIds) - keys.add(new BinaryMetadataKey(typeId)); - Map res = U.newHashMap(metadataLocCache.size()); for (Map.Entry e : metadataLocCache.entrySet()) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java index 232ce383e59f5..b5b193bec258b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java @@ -70,9 +70,6 @@ public class GridClientPartitionTopology implements GridDhtPartitionTopology { /** Flag to control amount of output for full map. */ private static final boolean FULL_MAP_DEBUG = false; - /** */ - private static final Long ZERO = 0L; - /** Cache shared context. */ private GridCacheSharedContext cctx; @@ -1041,8 +1038,12 @@ else if (owners.contains(e.getKey())) Map> res = U.newHashMap(cntrMap.size()); for (Map.Entry> e : cntrMap.entrySet()) { - if (!e.getValue().equals(ZERO)) - res.put(e.getKey(), e.getValue()); + T2 val = e.getValue(); + + if (val.get1() == 0L && val.get2() == 0L) + continue; + + res.put(e.getKey(), e.getValue()); } return res; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java index 0ea48e38c4fd4..8430f84ef4f94 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java @@ -181,13 +181,13 @@ void init() { } fut.listen(new CI1>() { - @Override public void apply(IgniteInternalFuture fut) { - try { - fut.get(); - } - catch (IgniteCheckedException e) { - if (log.isDebugEnabled()) - log.debug("Failed to request keys from preloader [keys=" + keys + ", err=" + e + ']'); + @Override public void apply(IgniteInternalFuture fut) { + try { + fut.get(); + } + catch (IgniteCheckedException e) { + if (log.isDebugEnabled()) + log.debug("Failed to request keys from preloader [keys=" + keys + ", err=" + e + ']'); onDone(e); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java index e67dabf79d668..712babd999333 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java @@ -3493,7 +3493,7 @@ private void processDhtAtomicDeferredUpdateResponse(UUID nodeId, GridDhtAtomicDe assert futIds != null && futIds.size() > 0 : futIds; for (int i = 0; i < futIds.size(); i++) { - Long id = futIds.get(i); + long id = futIds.get(i); GridDhtAtomicAbstractUpdateFuture updateFut = (GridDhtAtomicAbstractUpdateFuture)ctx.mvcc().atomicFuture(id); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalLockFuture.java index e2311b85e73f2..0197acf3ec59f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalLockFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalLockFuture.java @@ -447,11 +447,6 @@ private void onComplete(boolean success) { } } - /** {@inheritDoc} */ - @Override public int hashCode() { - return futId.hashCode(); - } - /** {@inheritDoc} */ @Override public String toString() { return S.toString(GridLocalLockFuture.class, this); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/io/PagesListMetaIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/io/PagesListMetaIO.java index 41e1bb51badca..1490a5c683a17 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/io/PagesListMetaIO.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/io/PagesListMetaIO.java @@ -151,7 +151,7 @@ public void getBucketsData(long pageAddr, Map res) { int off = offset(0); for (int i = 0; i < cnt; i++) { - Integer bucket = (int)PageUtils.getShort(pageAddr, off); + int bucket = (int)PageUtils.getShort(pageAddr, off); assert bucket >= 0 && bucket <= Short.MAX_VALUE : bucket; long tailId = PageUtils.getLong(pageAddr, off + 2); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java index c473bfe3f1324..34a9fc1c4f715 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java @@ -1502,39 +1502,39 @@ private void sendReply(UUID nodeId, GridDhtTxFinishRequest req, boolean committe entry.op() == TRANSFORM && entry.oldValueOnPrimary() && !entry.hasValue()) { - while (true) { - try { - GridCacheEntryEx cached = entry.cached(); - - if (cached == null) { - cached = cacheCtx.cache().entryEx(entry.key(), req.topologyVersion()); - - entry.cached(cached); - } - - CacheObject val = cached.innerGet( - /*ver*/null, - tx, - /*readThrough*/false, - /*updateMetrics*/false, - /*evt*/false, - tx.subjectId(), - /*transformClo*/null, - tx.resolveTaskName(), - /*expiryPlc*/null, - /*keepBinary*/true); - - if (val == null) - val = cacheCtx.toCacheObject(cacheCtx.store().load(null, entry.key())); - - if (val != null) - entry.readValue(val); - - break; - } - catch (GridCacheEntryRemovedException ignored) { - if (log.isDebugEnabled()) - log.debug("Got entry removed exception, will retry: " + entry.txKey()); + while (true) { + try { + GridCacheEntryEx cached = entry.cached(); + + if (cached == null) { + cached = cacheCtx.cache().entryEx(entry.key(), req.topologyVersion()); + + entry.cached(cached); + } + + CacheObject val = cached.innerGet( + /*ver*/null, + tx, + /*readThrough*/false, + /*updateMetrics*/false, + /*evt*/false, + tx.subjectId(), + /*transformClo*/null, + tx.resolveTaskName(), + /*expiryPlc*/null, + /*keepBinary*/true); + + if (val == null) + val = cacheCtx.toCacheObject(cacheCtx.store().load(null, entry.key())); + + if (val != null) + entry.readValue(val); + + break; + } + catch (GridCacheEntryRemovedException ignored) { + if (log.isDebugEnabled()) + log.debug("Got entry removed exception, will retry: " + entry.txKey()); entry.cached(cacheCtx.cache().entryEx(entry.key(), req.topologyVersion())); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java index 26a4a91201f19..e4e006156aad9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java @@ -2159,7 +2159,7 @@ private TxLocksResponse txLocksInfo(Collection txKeys) { } } // Special case for optimal sequence of nodes processing. - else if (nearTxLoc && requestedKeys != null && requestedKeys.contains(txKey.key())) { + else if (nearTxLoc && requestedKeys != null && requestedKeys.contains(txKey)) { TxLock txLock = new TxLock( tx.nearXidVersion(), tx.nodeId(), diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java index 42430741aad7b..e73fa6cd67a54 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java @@ -202,7 +202,7 @@ else if (msg instanceof IgfsAckMessage) grpSize = mapper instanceof IgfsGroupDataBlocksKeyMapper ? ((IgfsGroupDataBlocksKeyMapper)mapper).getGroupSize() : 1; - grpBlockSize = igfsCtx.configuration().getBlockSize() * grpSize; + grpBlockSize = igfsCtx.configuration().getBlockSize() * (long)grpSize; assert grpBlockSize != 0; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/redis/string/GridRedisIncrDecrCommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/redis/string/GridRedisIncrDecrCommandHandler.java index a57b82ebc24af..99453aa53d2fa 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/redis/string/GridRedisIncrDecrCommandHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/redis/string/GridRedisIncrDecrCommandHandler.java @@ -93,7 +93,7 @@ public GridRedisIncrDecrCommandHandler(final IgniteLogger log, final GridRestPro restReq.initial(0L); else { if (getResp.getResponse() instanceof String) { - Long init; + long init; try { init = Long.parseLong((String)getResp.getResponse()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java index 54ffe417b6ce1..acfd914a9225f 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java @@ -3380,18 +3380,18 @@ public static void writeStringToFile(File file, String s, String charset) throws * @throws IOException If error occurred. */ public static String readFileToString(String fileName, String charset) throws IOException { - Reader input = new InputStreamReader(new FileInputStream(fileName), charset); + try (Reader input = new InputStreamReader(new FileInputStream(fileName), charset)) { + StringWriter output = new StringWriter(); - StringWriter output = new StringWriter(); + char[] buf = new char[4096]; - char[] buf = new char[4096]; + int n; - int n; + while ((n = input.read(buf)) != -1) + output.write(buf, 0, n); - while ((n = input.read(buf)) != -1) - output.write(buf, 0, n); - - return output.toString(); + return output.toString(); + } } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java index ce7e7f3b6f52c..3b8b22c047a6e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java @@ -2129,14 +2129,16 @@ private void dumpStats(StringBuilder sb, int cnt = 0; - for (SessionWriteRequest req : ses.writeQueue()) { - Object msg = req.message(); + for (SessionWriteRequest req : ses.writeQueue()) { + Object msg = req.message(); if (shortInfo && msg instanceof GridIoMessage) - msg = ((GridIoMessage)msg).message().getClass().getSimpleName();if (cnt == 0) - sb.append(",\n opQueue=[").append(msg); - else - sb.append(',').append(msg); + msg = ((GridIoMessage)msg).message().getClass().getSimpleName(); + + if (cnt == 0) + sb.append(",\n opQueue=[").append(msg); + else + sb.append(',').append(msg); if (++cnt == 5) { sb.append(']'); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetricsCollectorTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetricsCollectorTask.java index 5cc8154c74c1d..8ce3c8c9b8f59 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetricsCollectorTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetricsCollectorTask.java @@ -94,7 +94,7 @@ private VisorCacheMetricsCollectorJob(VisorCacheMetricsCollectorTaskArg arg, boo @Override protected Collection run(final VisorCacheMetricsCollectorTaskArg arg) { assert arg != null; - Boolean showSysCaches = arg.isShowSystemCaches(); + boolean showSysCaches = arg.isShowSystemCaches(); Collection cacheNames = arg.getCacheNames(); diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java index 5b952e897640e..1b00b5d77218e 100755 --- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java @@ -3260,14 +3260,17 @@ else if (X.hasCause(e, SocketTimeoutException.class)) "operating system firewall is disabled on local and remote hosts) " + "[addrs=" + addrs + ']'); - if (enableForcibleNodeKill) {if (getSpiContext().node(node.id()) != null && (CU.clientNode(node) || !CU.clientNode(getLocalNode())) && - X.hasCause(errs, ConnectException.class,HandshakeException.class, SocketTimeoutException.class, HandshakeTimeoutException.class, - IgniteSpiOperationTimeoutException.class)) {String msg = "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " + + if (enableForcibleNodeKill) { + if (getSpiContext().node(node.id()) != null + && (CU.clientNode(node) || !CU.clientNode(getLocalNode())) && + connectionError(errs)) { + String msg = "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " + "cluster [" + "rmtNode=" + node + ']'; - if(enableTroubleshootingLog)U.error(log, msg, errs); + if (enableTroubleshootingLog) + U.error(log, msg, errs); else - U.warn(log, msg); + U.warn(log, msg); getSpiContext().failNode(node.id(), "TcpCommunicationSpi failed to establish connection to node [" + "rmtNode=" + node + @@ -3283,6 +3286,18 @@ else if (X.hasCause(e, SocketTimeoutException.class)) return client; } + /** + * @param errs Error. + * @return {@code True} if error was caused by some connection IO error. + */ + private static boolean connectionError(IgniteCheckedException errs) { + return X.hasCause(errs, ConnectException.class, + HandshakeException.class, + SocketTimeoutException.class, + HandshakeTimeoutException.class, + IgniteSpiOperationTimeoutException.class); + } + /** * Performs handshake in timeout-safe way. * diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java index d621fb3f5e9f9..e098787726ad7 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java @@ -6249,7 +6249,7 @@ private boolean processJoinRequestMessage(TcpDiscoveryJoinRequestMessage msg, else { spi.stats.onMessageProcessingStarted(msg); - Integer res; + int res; SocketAddress rmtAddr = sock.getRemoteSocketAddress(); diff --git a/modules/core/src/main/java/org/apache/ignite/spi/failover/always/AlwaysFailoverSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/failover/always/AlwaysFailoverSpi.java index 468a6275f2c54..f07411997bf6c 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/failover/always/AlwaysFailoverSpi.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/failover/always/AlwaysFailoverSpi.java @@ -237,7 +237,7 @@ public int getTotalFailoverJobsCount() { if (failedNodes == null) failedNodes = U.newHashSet(1); - Integer failoverCnt = failedNodes.size(); + int failoverCnt = failedNodes.size(); if (failoverCnt >= maxFailoverAttempts) { U.warn(log, "Job failover failed because number of maximum failover attempts is exceeded [failedJob=" + diff --git a/modules/mesos/src/main/java/org/apache/ignite/mesos/ClusterProperties.java b/modules/mesos/src/main/java/org/apache/ignite/mesos/ClusterProperties.java index 3c8f39645195c..3e66cc66bd65c 100644 --- a/modules/mesos/src/main/java/org/apache/ignite/mesos/ClusterProperties.java +++ b/modules/mesos/src/main/java/org/apache/ignite/mesos/ClusterProperties.java @@ -478,7 +478,9 @@ public static ClusterProperties from(String cfg) { if (cfg != null) { props = new Properties(); - props.load(new FileInputStream(cfg)); + try (FileInputStream in = new FileInputStream(cfg)) { + props.load(in); + } } ClusterProperties prop = new ClusterProperties(); diff --git a/modules/web/src/main/java/org/apache/ignite/cache/websession/WebSessionFilter.java b/modules/web/src/main/java/org/apache/ignite/cache/websession/WebSessionFilter.java index 0644c0f42bad7..6cbf575827676 100644 --- a/modules/web/src/main/java/org/apache/ignite/cache/websession/WebSessionFilter.java +++ b/modules/web/src/main/java/org/apache/ignite/cache/websession/WebSessionFilter.java @@ -740,7 +740,7 @@ private WebSessionV2 createSessionV2(HttpServletRequest httpReq) throws IOExcept private IgniteCache cacheWithExpiryPolicy(final int maxInactiveInteval, final IgniteCache cache) { if (maxInactiveInteval > 0) { - long ttl = maxInactiveInteval * 1000; + long ttl = maxInactiveInteval * 1000L; ExpiryPolicy plc = new ModifiedExpiryPolicy(new Duration(MILLISECONDS, ttl)); From 39fa8fae3ec80fd3a4c07f4610ab46817b75dd23 Mon Sep 17 00:00:00 2001 From: Dmitriy Shabalin Date: Tue, 25 Jul 2017 14:30:47 +0700 Subject: [PATCH 019/547] IGNITE-5812 Set width for dropdown as for element. --- modules/web-console/frontend/app/app.js | 8 ++-- .../AngularStrapSelect.decorator.js} | 2 +- .../AngularStrapTooltip.decorator.js} | 38 +++++++++++++++++-- .../configuration/domains-import.tpl.pug | 4 +- 4 files changed, 41 insertions(+), 11 deletions(-) rename modules/web-console/frontend/app/{decorator/select.js => services/AngularStrapSelect.decorator.js} (98%) rename modules/web-console/frontend/app/{decorator/tooltip.js => services/AngularStrapTooltip.decorator.js} (67%) diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js index dc5c6e9729567..0667e8fcdcfb5 100644 --- a/modules/web-console/frontend/app/app.js +++ b/modules/web-console/frontend/app/app.js @@ -20,9 +20,6 @@ import '../app/primitives'; import './app.config'; -import './decorator/select'; -import './decorator/tooltip'; - import './modules/form/form.module'; import './modules/agent/agent.module'; import './modules/sql/sql.module'; @@ -97,6 +94,9 @@ import UnsavedChangesGuard from './services/UnsavedChangesGuard.service'; import Clusters from './services/Clusters'; import Caches from './services/Caches'; +import AngularStrapTooltip from './services/AngularStrapTooltip.decorator'; +import AngularStrapSelect from './services/AngularStrapSelect.decorator'; + // Filters. import byName from './filters/byName.filter'; import defaultName from './filters/default-name.filter'; @@ -195,6 +195,8 @@ angular gridColumnSelector.name, bsSelectMenu.name, protectFromBsSelectRender.name, + AngularStrapTooltip.name, + AngularStrapSelect.name, // Ignite modules. IgniteModules.name ]) diff --git a/modules/web-console/frontend/app/decorator/select.js b/modules/web-console/frontend/app/services/AngularStrapSelect.decorator.js similarity index 98% rename from modules/web-console/frontend/app/decorator/select.js rename to modules/web-console/frontend/app/services/AngularStrapSelect.decorator.js index 2d22707d1e0fa..39f7ccdb6bec2 100644 --- a/modules/web-console/frontend/app/decorator/select.js +++ b/modules/web-console/frontend/app/services/AngularStrapSelect.decorator.js @@ -21,7 +21,7 @@ import angular from 'angular'; * Special decorator that fix problem in AngularStrap selectAll / deselectAll methods. * If this problem will be fixed in AngularStrap we can remove this delegate. */ -angular.module('mgcrea.ngStrap.select') +export default angular.module('mgcrea.ngStrap.select') .decorator('$select', ['$delegate', ($delegate) => { function SelectFactoryDecorated(element, controller, config) { const delegate = $delegate(element, controller, config); diff --git a/modules/web-console/frontend/app/decorator/tooltip.js b/modules/web-console/frontend/app/services/AngularStrapTooltip.decorator.js similarity index 67% rename from modules/web-console/frontend/app/decorator/tooltip.js rename to modules/web-console/frontend/app/services/AngularStrapTooltip.decorator.js index 71ea69478948a..d01a45075b794 100644 --- a/modules/web-console/frontend/app/decorator/tooltip.js +++ b/modules/web-console/frontend/app/services/AngularStrapTooltip.decorator.js @@ -16,13 +16,16 @@ */ import angular from 'angular'; +import flow from 'lodash/flow'; /** - * Special decorator that fix problem in AngularStrap $tooltip in special case. - * Case: when tooltip is shown on table row remove button and user click this button. - * If this problem will be fixed in AngularStrap we can remove this delegate. + * Decorator that fix problem in AngularStrap $tooltip. */ -angular.module('mgcrea.ngStrap.tooltip') +export default angular + .module('mgcrea.ngStrap.tooltip') + /** + * Don't hide tooltip when mouse move from element to tooltip. + */ .decorator('$tooltip', ['$delegate', ($delegate) => { function TooltipFactoryDecorated(element, config) { let tipElementEntered = false; @@ -70,4 +73,31 @@ angular.module('mgcrea.ngStrap.tooltip') } return TooltipFactoryDecorated; + }]) + /** + * Set width for dropdown as for element. + */ + .decorator('$tooltip', ['$delegate', ($delegate) => { + return function(el, config) { + const $tooltip = $delegate(el, config); + + $tooltip.$referenceElement = el; + $tooltip.destroy = flow($tooltip.destroy, () => $tooltip.$referenceElement = null); + $tooltip.$applyPlacement = flow($tooltip.$applyPlacement, () => { + if (!$tooltip.$element) + return; + + const refWidth = $tooltip.$referenceElement[0].getBoundingClientRect().width; + const elWidth = $tooltip.$element[0].getBoundingClientRect().width; + + if (refWidth > elWidth) { + $tooltip.$element.css({ + width: refWidth, + maxWidth: 'initial' + }); + } + }); + + return $tooltip; + }; }]); diff --git a/modules/web-console/frontend/views/configuration/domains-import.tpl.pug b/modules/web-console/frontend/views/configuration/domains-import.tpl.pug index 7097197e6671a..e9ed6f3c0b687 100644 --- a/modules/web-console/frontend/views/configuration/domains-import.tpl.pug +++ b/modules/web-console/frontend/views/configuration/domains-import.tpl.pug @@ -65,13 +65,11 @@ mixin td-ellipses-lbl(w, lbl) +ignite-form-field-dropdown('Driver JAR:', 'ui.selectedJdbcDriverJar', '"jdbcDriverJar"', false, true, false, 'Choose JDBC driver', '', 'jdbcDriverJars', 'Select appropriate JAR with JDBC driver
To add another driver you need to place it into "/jdbc-drivers" folder of Ignite Web Agent
Refer to Ignite Web Agent README.txt for for more information' - )( - data-ignite-form-field-input-autofocus='true' ) .settings-row.settings-row_small-label +java-class('JDBC driver:', 'selectedPreset.jdbcDriverClass', '"jdbcDriverClass"', true, true, 'Fully qualified class name of JDBC driver that will be used to connect to database') .settings-row.settings-row_small-label - +text('JDBC URL:', 'selectedPreset.jdbcUrl', '"jdbcUrl"', true, 'JDBC URL', 'JDBC URL for connecting to database
Refer to your database documentation for details') + +text-enabled-autofocus('JDBC URL:', 'selectedPreset.jdbcUrl', '"jdbcUrl"', true, true, 'JDBC URL', 'JDBC URL for connecting to database
Refer to your database documentation for details') .settings-row.settings-row_small-label +text('User:', 'selectedPreset.user', '"jdbcUser"', false, '', 'User name for connecting to database') .settings-row.settings-row_small-label From f277530ab6a9a6f03fd66f95a7a4d91a3506dcf5 Mon Sep 17 00:00:00 2001 From: Andrey Novikov Date: Tue, 25 Jul 2017 15:24:13 +0700 Subject: [PATCH 020/547] IGNITE-5697 Web Console: Upgrade dependencies. --- modules/web-console/backend/.eslintrc | 39 ++++++----- modules/web-console/backend/package.json | 27 ++++---- modules/web-console/frontend/.eslintrc | 40 +++++------ modules/web-console/frontend/app/app.js | 37 +++++------ .../activities-user-dialog/index.js | 28 ++++---- .../page-configure-advanced/controller.js | 9 +-- .../page-configure-basic/service.js | 8 +-- .../components/page-configure/controller.js | 12 +--- .../page-configure/controller.spec.js | 48 -------------- .../page-configure/services/PageConfigure.js | 18 ++--- .../web-console-header/component.js | 12 ++-- .../components/web-console-header/index.js | 4 +- .../frontend/app/modules/ace.module.js | 6 +- .../app/modules/branding/branding.module.js | 16 +++-- .../generator/ConfigurationGenerator.js | 2 +- .../generator/PlatformGenerator.js | 4 +- .../frontend/app/modules/demo/Demo.module.js | 10 +-- .../app/modules/dialog/dialog.controller.js | 4 -- .../app/modules/form/field/label.directive.js | 4 +- .../modules/form/field/tooltip.directive.js | 4 +- .../app/modules/sql/sql.controller.js | 2 +- .../frontend/app/modules/sql/sql.module.js | 66 +++++++++---------- .../app/modules/states/admin.state.js | 6 +- .../app/modules/states/configuration.state.js | 35 ++++++---- .../summary/summary-zipper.service.js | 2 +- .../app/modules/states/errors.state.js | 4 +- .../app/modules/states/logout.state.js | 12 ++-- .../app/modules/states/password.state.js | 4 +- .../app/modules/states/profile.state.js | 9 ++- .../app/modules/states/signin.state.js | 32 ++++----- .../app/modules/user/AclRoute.provider.js | 6 +- .../frontend/app/modules/user/user.module.js | 30 ++++++++- .../app/services/CopyToClipboard.service.js | 4 +- .../app/services/LegacyTable.service.js | 2 +- modules/web-console/frontend/app/vendor.js | 8 ++- modules/web-console/frontend/package.json | 57 ++++++++-------- modules/web-console/frontend/views/index.pug | 14 +--- 37 files changed, 292 insertions(+), 333 deletions(-) delete mode 100644 modules/web-console/frontend/app/components/page-configure/controller.spec.js diff --git a/modules/web-console/backend/.eslintrc b/modules/web-console/backend/.eslintrc index 7eb04b7964f53..1915518ab17b8 100644 --- a/modules/web-console/backend/.eslintrc +++ b/modules/web-console/backend/.eslintrc @@ -3,19 +3,21 @@ env: node: true mocha: true -ecmaFeatures: - arrowFunctions: true - blockBindings: true - classes: true - defaultParams: true - destructuring: true - module: true - objectLiteralComputedProperties: true - objectLiteralShorthandMethods: true - objectLiteralShorthandProperties: true - spread: true - templateStrings: true - experimentalObjectRestSpread: true +parserOptions: + sourceType: module + ecmaFeatures: + arrowFunctions: true + blockBindings: true + classes: true + defaultParams: true + destructuring: true + module: true + objectLiteralComputedProperties: true + objectLiteralShorthandMethods: true + objectLiteralShorthandProperties: true + spread: true + templateStrings: true + experimentalObjectRestSpread: true globals: _: true @@ -47,7 +49,7 @@ rules: guard-for-in: 1 handle-callback-err: 0 id-length: [2, {"min": 1, "max": 60}] - indent: [2, 4, {"SwitchCase": 1}] + indent: [2, 4, {"SwitchCase": 1, "MemberExpression": "off", "CallExpression": {"arguments": "off"}}] key-spacing: [2, { "beforeColon": false, "afterColon": true }] lines-around-comment: 0 linebreak-style: [0, "unix"] @@ -104,7 +106,7 @@ rules: no-loop-func: 2 no-mixed-requires: [0, false] no-mixed-spaces-and-tabs: [2, true] - no-multi-spaces: 2 + no-multi-spaces: ["error", {"exceptions": { "VariableDeclarator": true }}] no-multi-str: 2 no-multiple-empty-lines: [0, {"max": 2}] no-native-reassign: 2 @@ -130,7 +132,7 @@ rules: no-script-url: 0 no-self-compare: 2 no-sequences: 2 - no-shadow: 2 + no-shadow: 0 no-shadow-restricted-names: 2 no-spaced-func: 2 no-sparse-arrays: 1 @@ -162,7 +164,7 @@ rules: prefer-const: 1 prefer-spread: 2 quote-props: [2, "as-needed"] - quotes: [2, "single"] + quotes: [2, "single", {"allowTemplateLiterals": true}] radix: 1 semi: [2, "always"] semi-spacing: [2, {"before": false, "after": true}] @@ -181,6 +183,3 @@ rules: wrap-iife: 0 wrap-regex: 0 yoda: [2, "never"] - -parserOptions: - sourceType: module diff --git a/modules/web-console/backend/package.json b/modules/web-console/backend/package.json index 2af7787c8ca98..837f41c7e1495 100644 --- a/modules/web-console/backend/package.json +++ b/modules/web-console/backend/package.json @@ -29,19 +29,19 @@ "win32" ], "dependencies": { - "body-parser": "1.17.1", + "body-parser": "1.17.2", "connect-mongo": "1.3.2", - "cookie-parser": "~1.4.0", - "express": "4.15.2", - "express-session": "1.15.2", + "cookie-parser": "1.4.3", + "express": "4.15.3", + "express-session": "1.15.4", "fire-up": "1.0.0", - "glob": "7.1.1", + "glob": "7.1.2", "jszip": "3.1.3", "lodash": "4.17.4", - "mongoose": "4.9.4", - "morgan": "1.8.1", + "mongoose": "4.11.4", + "morgan": "1.8.2", "nconf": "0.8.4", - "nodemailer": "3.1.4", + "nodemailer": "4.0.1", "passport": "0.3.2", "passport-local": "1.0.0", "passport-local-mongoose": "4.0.0", @@ -49,12 +49,11 @@ "socket.io": "1.7.3" }, "devDependencies": { - "chai": "3.5.0", - "cross-env": "4.0.0", - "eslint": "3.19.0", - "eslint-friendly-formatter": "2.0.7", - "jasmine-core": "2.5.2", - "mocha": "3.2.0", + "chai": "4.1.0", + "cross-env": "5.0.1", + "eslint": "4.3.0", + "eslint-friendly-formatter": "3.0.0", + "mocha": "3.4.2", "mocha-teamcity-reporter": "1.1.1", "mockgoose": "6.0.8", "supertest": "3.0.0" diff --git a/modules/web-console/frontend/.eslintrc b/modules/web-console/frontend/.eslintrc index d88fb976af16e..4e24d0b75b7a7 100644 --- a/modules/web-console/frontend/.eslintrc +++ b/modules/web-console/frontend/.eslintrc @@ -6,20 +6,21 @@ plugins: env: es6: true browser: true - -ecmaFeatures: - arrowFunctions: true - blockBindings: true - classes: true - defaultParams: true - destructuring: true - module: true - objectLiteralComputedProperties: true - objectLiteralShorthandMethods: true - objectLiteralShorthandProperties: true - spread: true - templateStrings: true - experimentalObjectRestSpread: true +parserOptions: + sourceType: module + ecmaFeatures: + arrowFunctions: true + blockBindings: true + classes: true + defaultParams: true + destructuring: true + module: true + objectLiteralComputedProperties: true + objectLiteralShorthandMethods: true + objectLiteralShorthandProperties: true + spread: true + templateStrings: true + experimentalObjectRestSpread: true globals: _: true @@ -29,10 +30,6 @@ globals: window: true global: true angular: true - $generatorCommon: true - $generatorSpring: true - $generatorJava: true - $generatorOptional: true saveAs: true process: true require: true @@ -63,7 +60,7 @@ rules: guard-for-in: 1 handle-callback-err: 0 id-length: [2, {"min": 1, "max": 60}] - indent: [2, 4, {"SwitchCase": 1}] + indent: [2, 4, {"SwitchCase": 1, "MemberExpression": "off", "CallExpression": {"arguments": "off"}}] key-spacing: [2, { "beforeColon": false, "afterColon": true }] lines-around-comment: 0 linebreak-style: [0, "unix"] @@ -138,7 +135,7 @@ rules: no-path-concat: 0 no-plusplus: 0 no-process-env: 0 - no-process-exit: 1 + no-process-exit: 0 no-proto: 2 no-redeclare: 2 no-regex-spaces: 1 @@ -198,6 +195,3 @@ rules: wrap-regex: 0 yoda: [2, "never"] babel/semi: 2 - -parserOptions: - sourceType: module diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js index 0667e8fcdcfb5..e2d609a66141c 100644 --- a/modules/web-console/frontend/app/app.js +++ b/modules/web-console/frontend/app/app.js @@ -133,8 +133,7 @@ import IgniteModules from 'IgniteModules/index'; import baseTemplate from 'views/base.pug'; -angular -.module('ignite-console', [ +angular.module('ignite-console', [ // Optional AngularJS modules. 'ngAnimate', 'ngSanitize', @@ -143,6 +142,7 @@ angular 'btford.socket-io', 'mgcrea.ngStrap', 'ui.router', + 'ui.router.state.events', 'gridster', 'dndLists', 'nvd3', @@ -278,33 +278,32 @@ angular $urlRouterProvider.otherwise('/404'); $locationProvider.html5Mode(true); }]) -.run(['$rootScope', '$state', 'MetaTags', 'gettingStarted', ($root, $state, $meta, gettingStarted) => { +.run(['$rootScope', '$state', 'gettingStarted', ($root, $state, gettingStarted) => { $root._ = _; $root.$state = $state; - $root.$meta = $meta; $root.gettingStarted = gettingStarted; }]) .run(['$rootScope', 'AgentManager', ($root, agentMgr) => { $root.$on('user', () => agentMgr.connect()); }]) -.run(['$rootScope', ($root) => { - $root.$on('$stateChangeStart', () => { +.run(['$transitions', ($transitions) => { + $transitions.onStart({ }, () => { _.forEach(angular.element('.modal'), (m) => angular.element(m).scope().$hide()); }); - if (!$root.IgniteDemoMode) { - $root.$on('$stateChangeSuccess', (event, {name, unsaved}, params) => { - try { - if (unsaved) - localStorage.removeItem('lastStateChangeSuccess'); - else - localStorage.setItem('lastStateChangeSuccess', JSON.stringify({name, params})); - } - catch (ignored) { - // No-op. - } - }); - } + $transitions.onSuccess({ }, (trans) => { + try { + const {name, params, unsaved} = trans.$to(); + + if (unsaved) + localStorage.removeItem('lastStateChangeSuccess'); + else + localStorage.setItem('lastStateChangeSuccess', JSON.stringify({name, params})); + } + catch (ignored) { + // No-op. + } + }); }]) .run(['$rootScope', '$http', '$state', 'IgniteMessages', 'User', 'IgniteNotebookData', ($root, $http, $state, Messages, User, Notebook) => { // eslint-disable-line no-shadow diff --git a/modules/web-console/frontend/app/components/activities-user-dialog/index.js b/modules/web-console/frontend/app/components/activities-user-dialog/index.js index 02c7c1e0e9efe..13c1d95c74c61 100644 --- a/modules/web-console/frontend/app/components/activities-user-dialog/index.js +++ b/modules/web-console/frontend/app/components/activities-user-dialog/index.js @@ -15,20 +15,20 @@ * limitations under the License. */ - import controller from './activities-user-dialog.controller'; - import templateUrl from './activities-user-dialog.tpl.pug'; +import controller from './activities-user-dialog.controller'; +import templateUrl from './activities-user-dialog.tpl.pug'; - export default ['$modal', ($modal) => ({ show = true, user }) => { - const ActivitiesUserDialog = $modal({ - templateUrl, - show, - resolve: { - user: () => user - }, - controller, - controllerAs: 'ctrl' - }); +export default ['$modal', ($modal) => ({ show = true, user }) => { + const ActivitiesUserDialog = $modal({ + templateUrl, + show, + resolve: { + user: () => user + }, + controller, + controllerAs: 'ctrl' + }); - return ActivitiesUserDialog.$promise + return ActivitiesUserDialog.$promise .then(() => ActivitiesUserDialog); - }]; +}]; diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/controller.js index 0e73ae3a4b833..da2052705869e 100644 --- a/modules/web-console/frontend/app/components/page-configure-advanced/controller.js +++ b/modules/web-console/frontend/app/components/page-configure-advanced/controller.js @@ -16,7 +16,7 @@ */ export default class PageConfigureAdvancedController { - static $inject = ['PageConfigureAdvanced', '$scope']; + static $inject = ['$scope']; static menuItems = [ { text: 'Clusters', sref: 'base.configuration.tabs.advanced.clusters' }, @@ -26,14 +26,11 @@ export default class PageConfigureAdvancedController { { text: 'Summary', sref: 'base.configuration.tabs.advanced.summary' } ]; - constructor(PageConfigureAdvanced, $scope) { - Object.assign(this, {PageConfigureAdvanced, $scope}); + constructor($scope) { + Object.assign(this, {$scope}); } $onInit() { this.menuItems = this.constructor.menuItems; - this.$scope.$on('$stateChangeSuccess', (e, toState) => { - this.PageConfigureAdvanced.onStateEnterRedirect(toState); - }); } } diff --git a/modules/web-console/frontend/app/components/page-configure-basic/service.js b/modules/web-console/frontend/app/components/page-configure-basic/service.js index 4db61eaefa0c8..00321063ab3d5 100644 --- a/modules/web-console/frontend/app/components/page-configure-basic/service.js +++ b/modules/web-console/frontend/app/components/page-configure-basic/service.js @@ -50,7 +50,7 @@ export default class PageConfigureBasic { const noFakeIDCaches = caches.map(stripFakeID); cluster = cloneDeep(stripFakeID(cluster)); return this.$q.all(noFakeIDCaches.map((cache) => ( - this.caches.saveCache(cache) + this.caches.saveCache(cache) .then( ({data}) => data, (e) => { @@ -58,7 +58,7 @@ export default class PageConfigureBasic { return this.$q.resolve(null); } ) - ))) + ))) .then((cacheIDs) => { // Make sure we don't loose new IDs even if some requests fail this.pageConfigure.upsertCaches( @@ -115,8 +115,8 @@ export default class PageConfigureBasic { setCluster(_id) { this.ConfigureState.dispatchAction( isNewItem({_id}) - ? {type: SET_CLUSTER, _id, cluster: this.clusters.getBlankCluster()} - : {type: SET_CLUSTER, _id} + ? {type: SET_CLUSTER, _id, cluster: this.clusters.getBlankCluster()} + : {type: SET_CLUSTER, _id} ); } diff --git a/modules/web-console/frontend/app/components/page-configure/controller.js b/modules/web-console/frontend/app/components/page-configure/controller.js index 3dcd14e87eba6..5ead0bb08d241 100644 --- a/modules/web-console/frontend/app/components/page-configure/controller.js +++ b/modules/web-console/frontend/app/components/page-configure/controller.js @@ -16,15 +16,9 @@ */ export default class PageConfigureController { - static $inject = ['$scope', 'PageConfigure']; + static $inject = ['$scope']; - constructor($scope, PageConfigure) { - Object.assign(this, {$scope, PageConfigure}); - } - - $onInit() { - this.$scope.$on('$stateChangeSuccess', (e, toState) => { - this.PageConfigure.onStateEnterRedirect(toState); - }); + constructor($scope) { + Object.assign(this, {$scope}); } } diff --git a/modules/web-console/frontend/app/components/page-configure/controller.spec.js b/modules/web-console/frontend/app/components/page-configure/controller.spec.js deleted file mode 100644 index e30eb65665f55..0000000000000 --- a/modules/web-console/frontend/app/components/page-configure/controller.spec.js +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import {suite, test} from 'mocha'; -import {assert} from 'chai'; -import {spy} from 'sinon'; - -import Controller from './controller'; - -const mocks = () => new Map([ - ['$scope', { - $on: spy() - }], - ['PageConfigure', { - onStateEnterRedirect: spy() - }] -]); - -suite('page-configure component controller', () => { - test('State change success redirect', () => { - const c = new Controller(...mocks().values()); - c.$onInit(); - c.$scope.$on.getCall(0).args[1](null, {name: 'base.items'}); - assert.isOk( - c.PageConfigure.onStateEnterRedirect.calledOnce, - 'calls PageConfigure.onStateEnterRedirect every $stateChangeSuccess' - ); - assert.deepEqual( - c.PageConfigure.onStateEnterRedirect.getCall(0).args, - [{name: 'base.items'}], - 'calls PageConfigure.onStateEnterRedirect with correct arguments' - ); - }); -}); diff --git a/modules/web-console/frontend/app/components/page-configure/services/PageConfigure.js b/modules/web-console/frontend/app/components/page-configure/services/PageConfigure.js index a712aca701e3d..34a292a0b25e7 100644 --- a/modules/web-console/frontend/app/components/page-configure/services/PageConfigure.js +++ b/modules/web-console/frontend/app/components/page-configure/services/PageConfigure.js @@ -31,15 +31,17 @@ export default class PageConfigure { } onStateEnterRedirect(toState) { - if (toState.name !== 'base.configuration.tabs') return this.$q.resolve(); + if (toState.name !== 'base.configuration.tabs') + return this.$q.resolve(); + return this.configuration.read() - .then((data) => { - this.loadList(data); - const nextState = data.clusters.length - ? 'base.configuration.tabs.advanced' - : 'base.configuration.tabs.basic'; - return this.$state.go(nextState, null, {location: 'replace'}); - }); + .then((data) => { + this.loadList(data); + + return this.$q.resolve(data.clusters.length + ? 'base.configuration.tabs.advanced' + : 'base.configuration.tabs.basic'); + }); } loadList(list) { diff --git a/modules/web-console/frontend/app/components/web-console-header/component.js b/modules/web-console/frontend/app/components/web-console-header/component.js index d4c5c4bd65bbd..66f09e9c71a2f 100644 --- a/modules/web-console/frontend/app/components/web-console-header/component.js +++ b/modules/web-console/frontend/app/components/web-console-header/component.js @@ -33,11 +33,15 @@ export default { Object.assign(this, {$rootScope, $scope, $state, branding, UserNotifications}); } + setWebAgentDownloadVisible() { + this.isWebAgentDownloadVisible = + this.constructor.webAgentDownloadVisibleStates.some((state) => this.$state.includes(state)); + } + $onInit() { - this.$scope.$on('$stateChangeSuccess', () => { - this.isWebAgentDownloadVisible = - this.constructor.webAgentDownloadVisibleStates.some((state) => this.$state.includes(state)); - }); + this.setWebAgentDownloadVisible(); + + this.$scope.$on('$stateChangeSuccess', () => this.setWebAgentDownloadVisible()); } }, transclude: { diff --git a/modules/web-console/frontend/app/components/web-console-header/index.js b/modules/web-console/frontend/app/components/web-console-header/index.js index 36caa3dfed876..e41e1ccc273e6 100644 --- a/modules/web-console/frontend/app/components/web-console-header/index.js +++ b/modules/web-console/frontend/app/components/web-console-header/index.js @@ -19,5 +19,5 @@ import angular from 'angular'; import component from './component'; export default angular - .module('ignite-console.web-console-header', []) - .component('webConsoleHeader', component); + .module('ignite-console.web-console-header', []) + .component('webConsoleHeader', component); diff --git a/modules/web-console/frontend/app/modules/ace.module.js b/modules/web-console/frontend/app/modules/ace.module.js index 4920a6fc11c97..a28536a0233fa 100644 --- a/modules/web-console/frontend/app/modules/ace.module.js +++ b/modules/web-console/frontend/app/modules/ace.module.js @@ -184,10 +184,10 @@ angular return (e) => { const newValue = session.getValue(); + // HACK make sure to only trigger the apply outside of the + // digest loop 'cause ACE is actually using this callback + // for any text transformation ! if (ngModel && newValue !== ngModel.$viewValue && - // HACK make sure to only trigger the apply outside of the - // digest loop 'cause ACE is actually using this callback - // for any text transformation ! !scope.$$phase && !scope.$root.$$phase) scope.$eval(() => ngModel.$setViewValue(newValue)); diff --git a/modules/web-console/frontend/app/modules/branding/branding.module.js b/modules/web-console/frontend/app/modules/branding/branding.module.js index cae7c917c43f8..9c4a5e57d3c74 100644 --- a/modules/web-console/frontend/app/modules/branding/branding.module.js +++ b/modules/web-console/frontend/app/modules/branding/branding.module.js @@ -28,14 +28,18 @@ import ignitePoweredByApache from './powered-by-apache.directive'; angular .module('ignite-console.branding', [ - 'ui.router.metatags' + 'tf.metatags' ]) .provider(...IgniteBranding) -.config(['UIRouterMetatagsProvider', (UIRouterMetatagsProvider) => { - UIRouterMetatagsProvider - .setDefaultTitle('Apache Ignite - Management Tool and Configuration Wizard') - .setTitleSuffix(' – Apache Ignite Web Console') - .setDefaultDescription('The Apache Ignite Web Console is an interactive management tool and configuration wizard which walks you through the creation of config files. Try it now.'); +.config(['tfMetaTagsProvider', (tfMetaTagsProvider) => { + tfMetaTagsProvider.setDefaults({ + title: 'Apache Ignite - Management Tool and Configuration Wizard', + properties: { + description: 'The Apache Ignite Web Console is an interactive management tool and configuration wizard which walks you through the creation of config files. Try it now.' + } + }); + + tfMetaTagsProvider.setTitleSuffix(' – Apache Ignite Web Console'); }]) .directive(...ignitePoweredByApache) .directive(...igniteHeaderLogo) diff --git a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js index 8299b9bb7a1c0..a14bfd339c0b4 100644 --- a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js +++ b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js @@ -887,7 +887,7 @@ export default class IgniteConfigurationGenerator { cfg.intProperty('peerClassLoadingMissedResourcesCacheSize') .intProperty('peerClassLoadingThreadPoolSize') .varArgProperty('p2pLocClsPathExcl', 'peerClassLoadingLocalClassPathExclude', - cluster.peerClassLoadingLocalClassPathExclude); + cluster.peerClassLoadingLocalClassPathExclude); } // Since ignite 2.0 diff --git a/modules/web-console/frontend/app/modules/configuration/generator/PlatformGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/PlatformGenerator.js index b07619331715e..234c7ecebfb56 100644 --- a/modules/web-console/frontend/app/modules/configuration/generator/PlatformGenerator.js +++ b/modules/web-console/frontend/app/modules/configuration/generator/PlatformGenerator.js @@ -168,7 +168,7 @@ export default ['JavaTypes', 'igniteClusterPlatformDefaults', 'igniteCachePlatfo .intProperty('unacknowledgedMessagesBufferSize') // .intProperty('socketWriteTimeout') .intProperty('selectorsCount'); - // .emptyBeanProperty('addressResolver'); + // .emptyBeanProperty('addressResolver'); if (commSpi.nonEmpty()) cfg.beanProperty('CommunicationSpi', commSpi); @@ -176,7 +176,7 @@ export default ['JavaTypes', 'igniteClusterPlatformDefaults', 'igniteCachePlatfo cfg.intProperty('networkTimeout', 'NetworkTimeout') .intProperty('networkSendRetryDelay') .intProperty('networkSendRetryCount'); - // .intProperty('discoveryStartupDelay'); + // .intProperty('discoveryStartupDelay'); return cfg; } diff --git a/modules/web-console/frontend/app/modules/demo/Demo.module.js b/modules/web-console/frontend/app/modules/demo/Demo.module.js index d75beda353974..24454613d6c05 100644 --- a/modules/web-console/frontend/app/modules/demo/Demo.module.js +++ b/modules/web-console/frontend/app/modules/demo/Demo.module.js @@ -24,7 +24,7 @@ angular .module('ignite-console.demo', [ 'ignite-console.socket' ]) -.config(['$stateProvider', 'AclRouteProvider', ($stateProvider, AclRoute) => { +.config(['$stateProvider', ($stateProvider) => { $stateProvider .state('demo', { abstract: true, @@ -33,17 +33,17 @@ angular }) .state('demo.resume', { url: '/resume', - onEnter: AclRoute.checkAccess('demo'), + permission: 'demo', controller: ['$state', ($state) => { $state.go('base.configuration.tabs.advanced.clusters'); }], - metaTags: { + tfMetaTags: { title: 'Demo resume' } }) .state('demo.reset', { url: '/reset', - onEnter: AclRoute.checkAccess('demo'), + permission: 'demo', controller: ['$state', '$http', 'IgniteMessages', ($state, $http, Messages) => { $http.post('/api/v1/demo/reset') .then(() => $state.go('base.configuration.tabs.advanced.clusters')) @@ -53,7 +53,7 @@ angular Messages.showError(res); }); }], - metaTags: { + tfMetaTags: { title: 'Demo reset' } }); diff --git a/modules/web-console/frontend/app/modules/dialog/dialog.controller.js b/modules/web-console/frontend/app/modules/dialog/dialog.controller.js index 05518d3132552..0256b84e4753a 100644 --- a/modules/web-console/frontend/app/modules/dialog/dialog.controller.js +++ b/modules/web-console/frontend/app/modules/dialog/dialog.controller.js @@ -33,8 +33,4 @@ export default ['$rootScope', '$scope', 'IgniteDialog', function($root, $scope, $scope.$watch(() => ctrl.content, () => { $scope.content = ctrl.content; }); - - $root.$on('$stateChangeStart', () => { - dialog.hide(); - }); }]; diff --git a/modules/web-console/frontend/app/modules/form/field/label.directive.js b/modules/web-console/frontend/app/modules/form/field/label.directive.js index 97ba5987f859f..94f7889107270 100644 --- a/modules/web-console/frontend/app/modules/form/field/label.directive.js +++ b/modules/web-console/frontend/app/modules/form/field/label.directive.js @@ -20,7 +20,7 @@ export default ['igniteFormFieldLabel', [() => { restrict: 'E', compile() { return { - post($scope, $element, $attrs, [form, field], $transclude) { + post($scope, $element, $attrs, [field], $transclude) { $transclude($scope, function(clone) { const text = clone.text(); @@ -42,6 +42,6 @@ export default ['igniteFormFieldLabel', [() => { }, replace: true, transclude: true, - require: ['^form', '?^igniteFormField'] + require: ['?^igniteFormField'] }; }]]; diff --git a/modules/web-console/frontend/app/modules/form/field/tooltip.directive.js b/modules/web-console/frontend/app/modules/form/field/tooltip.directive.js index 4f440a1b9875c..9e764bcfd0901 100644 --- a/modules/web-console/frontend/app/modules/form/field/tooltip.directive.js +++ b/modules/web-console/frontend/app/modules/form/field/tooltip.directive.js @@ -18,7 +18,7 @@ const template = ''; export default ['igniteFormFieldTooltip', ['$tooltip', ($tooltip) => { - const link = ($scope, $element, $attrs, [form, field], $transclude) => { + const link = ($scope, $element, $attrs, [field], $transclude) => { const content = Array.prototype.slice .apply($transclude($scope)) .reduce((html, el) => html += el.outerHTML || el.textContent || el, ''); @@ -44,6 +44,6 @@ export default ['igniteFormFieldTooltip', ['$tooltip', ($tooltip) => { link, replace: true, transclude: true, - require: ['^form', '?^igniteFormField'] + require: ['?^igniteFormField'] }; }]]; diff --git a/modules/web-console/frontend/app/modules/sql/sql.controller.js b/modules/web-console/frontend/app/modules/sql/sql.controller.js index 633f167684e78..3d3863415354a 100644 --- a/modules/web-console/frontend/app/modules/sql/sql.controller.js +++ b/modules/web-console/frontend/app/modules/sql/sql.controller.js @@ -1247,7 +1247,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval', const chartHistory = paragraph.chartHistory; - // Clear history on query change. + // Clear history on query change. if (clearChart) { chartHistory.length = 0; diff --git a/modules/web-console/frontend/app/modules/sql/sql.module.js b/modules/web-console/frontend/app/modules/sql/sql.module.js index 79614c2c9c73d..da9955c163239 100644 --- a/modules/web-console/frontend/app/modules/sql/sql.module.js +++ b/modules/web-console/frontend/app/modules/sql/sql.module.js @@ -27,37 +27,35 @@ import sqlTplUrl from 'app/../views/sql/sql.tpl.pug'; angular.module('ignite-console.sql', [ 'ui.router' ]) - .config(['$stateProvider', 'AclRouteProvider', - ($stateProvider, AclRoute) => { - // set up the states - $stateProvider - .state('base.sql', { - url: '/queries', - abstract: true, - template: '' - }) - .state('base.sql.notebook', { - url: '/notebook/{noteId}', - templateUrl: sqlTplUrl, - onEnter: AclRoute.checkAccess('query'), - metaTags: { - title: 'Query notebook' - }, - controller, - controllerAs: '$ctrl' - }) - .state('base.sql.demo', { - url: '/demo', - templateUrl: sqlTplUrl, - onEnter: AclRoute.checkAccess('query'), - metaTags: { - title: 'SQL demo' - }, - controller, - controllerAs: '$ctrl' - }); - }] - ) - .service('IgniteNotebookData', NotebookData) - .service('IgniteNotebook', Notebook) - .controller('notebookController', notebook); +.config(['$stateProvider', ($stateProvider) => { + // set up the states + $stateProvider + .state('base.sql', { + url: '/queries', + abstract: true, + template: '' + }) + .state('base.sql.notebook', { + url: '/notebook/{noteId}', + templateUrl: sqlTplUrl, + permission: 'query', + tfMetaTags: { + title: 'Query notebook' + }, + controller, + controllerAs: '$ctrl' + }) + .state('base.sql.demo', { + url: '/demo', + templateUrl: sqlTplUrl, + permission: 'query', + tfMetaTags: { + title: 'SQL demo' + }, + controller, + controllerAs: '$ctrl' + }); +}]) +.service('IgniteNotebookData', NotebookData) +.service('IgniteNotebook', Notebook) +.controller('notebookController', notebook); diff --git a/modules/web-console/frontend/app/modules/states/admin.state.js b/modules/web-console/frontend/app/modules/states/admin.state.js index 7189508b574bb..65e13aa98fd81 100644 --- a/modules/web-console/frontend/app/modules/states/admin.state.js +++ b/modules/web-console/frontend/app/modules/states/admin.state.js @@ -24,7 +24,7 @@ angular .module('ignite-console.states.admin', [ 'ui.router' ]) -.config(['$stateProvider', 'AclRouteProvider', function($stateProvider, AclRoute) { +.config(['$stateProvider', function($stateProvider) { // set up the states $stateProvider .state('base.settings.admin', { @@ -50,8 +50,8 @@ angular } }, // templateUrl, - onEnter: AclRoute.checkAccess('admin_page'), - metaTags: { + permission: 'admin_page', + tfMetaTags: { title: 'Admin panel' } }); diff --git a/modules/web-console/frontend/app/modules/states/configuration.state.js b/modules/web-console/frontend/app/modules/states/configuration.state.js index faf931500261b..d2c1410a9e7d9 100644 --- a/modules/web-console/frontend/app/modules/states/configuration.state.js +++ b/modules/web-console/frontend/app/modules/states/configuration.state.js @@ -51,7 +51,7 @@ angular.module('ignite-console.states.configuration', ['ui.router']) $templateCache.put('summary-tabs.html', summaryTabsTemplateUrl); }]) // Configure state provider. - .config(['$stateProvider', 'AclRouteProvider', ($stateProvider, AclRoute) => { + .config(['$stateProvider', ($stateProvider) => { // Setup the states. $stateProvider .state('base.configuration', { @@ -64,15 +64,21 @@ angular.module('ignite-console.states.configuration', ['ui.router']) }) .state('base.configuration.tabs', { url: '/configuration', + permission: 'configuration', template: '', - metaTags: { + redirectTo: (trans) => { + const PageConfigure = trans.injector().get('PageConfigure'); + + return PageConfigure.onStateEnterRedirect(trans.to()); + }, + tfMetaTags: { title: 'Configuration' } }) .state('base.configuration.tabs.basic', { url: '/basic', template: '', - metaTags: { + tfMetaTags: { title: 'Basic Configuration' }, resolve: { @@ -88,13 +94,14 @@ angular.module('ignite-console.states.configuration', ['ui.router']) }) .state('base.configuration.tabs.advanced', { url: '/advanced', - template: '' + template: '', + redirectTo: 'base.configuration.tabs.advanced.clusters' }) .state('base.configuration.tabs.advanced.clusters', { url: '/clusters', templateUrl: clustersTpl, - onEnter: AclRoute.checkAccess('configuration'), - metaTags: { + permission: 'configuration', + tfMetaTags: { title: 'Configure Clusters' }, controller: clustersCtrl, @@ -103,8 +110,8 @@ angular.module('ignite-console.states.configuration', ['ui.router']) .state('base.configuration.tabs.advanced.caches', { url: '/caches', templateUrl: cachesTpl, - onEnter: AclRoute.checkAccess('configuration'), - metaTags: { + permission: 'configuration', + tfMetaTags: { title: 'Configure Caches' }, controller: cachesCtrl, @@ -113,8 +120,8 @@ angular.module('ignite-console.states.configuration', ['ui.router']) .state('base.configuration.tabs.advanced.domains', { url: '/domains', templateUrl: domainsTpl, - onEnter: AclRoute.checkAccess('configuration'), - metaTags: { + permission: 'configuration', + tfMetaTags: { title: 'Configure Domain Model' }, controller: domainsCtrl, @@ -123,8 +130,8 @@ angular.module('ignite-console.states.configuration', ['ui.router']) .state('base.configuration.tabs.advanced.igfs', { url: '/igfs', templateUrl: igfsTpl, - onEnter: AclRoute.checkAccess('configuration'), - metaTags: { + permission: 'configuration', + tfMetaTags: { title: 'Configure IGFS' }, controller: igfsCtrl, @@ -133,10 +140,10 @@ angular.module('ignite-console.states.configuration', ['ui.router']) .state('base.configuration.tabs.advanced.summary', { url: '/summary', templateUrl: summaryTpl, - onEnter: AclRoute.checkAccess('configuration'), + permission: 'configuration', controller: ConfigurationSummaryCtrl, controllerAs: 'ctrl', - metaTags: { + tfMetaTags: { title: 'Configurations Summary' } }); diff --git a/modules/web-console/frontend/app/modules/states/configuration/summary/summary-zipper.service.js b/modules/web-console/frontend/app/modules/states/configuration/summary/summary-zipper.service.js index 0259d393045f0..47ce9ad427c0a 100644 --- a/modules/web-console/frontend/app/modules/states/configuration/summary/summary-zipper.service.js +++ b/modules/web-console/frontend/app/modules/states/configuration/summary/summary-zipper.service.js @@ -15,7 +15,7 @@ * limitations under the License. */ -import Worker from 'worker?inline=true!./summary.worker'; +import Worker from 'worker!./summary.worker'; export default ['$q', function($q) { return function(message) { diff --git a/modules/web-console/frontend/app/modules/states/errors.state.js b/modules/web-console/frontend/app/modules/states/errors.state.js index e3d4d415811ee..3cc03be9a3c22 100644 --- a/modules/web-console/frontend/app/modules/states/errors.state.js +++ b/modules/web-console/frontend/app/modules/states/errors.state.js @@ -29,7 +29,7 @@ angular .state('404', { url: '/404', templateUrl: templateNotFoundPage, - metaTags: { + tfMetaTags: { title: 'Page not found' }, unsaved: true @@ -37,7 +37,7 @@ angular .state('403', { url: '/403', templateUrl: templateNotAuthorizedPage, - metaTags: { + tfMetaTags: { title: 'Not authorized' }, unsaved: true diff --git a/modules/web-console/frontend/app/modules/states/logout.state.js b/modules/web-console/frontend/app/modules/states/logout.state.js index 42795ea46a41f..9f9c7c6774842 100644 --- a/modules/web-console/frontend/app/modules/states/logout.state.js +++ b/modules/web-console/frontend/app/modules/states/logout.state.js @@ -17,18 +17,16 @@ import angular from 'angular'; -angular -.module('ignite-console.states.logout', [ +angular.module('ignite-console.states.logout', [ 'ui.router' ]) -.config(['$stateProvider', 'AclRouteProvider', function($stateProvider, AclRoute) { +.config(['$stateProvider', function($stateProvider) { // set up the states - $stateProvider - .state('logout', { + $stateProvider.state('logout', { url: '/logout', - onEnter: AclRoute.checkAccess('logout'), + permission: 'logout', controller: ['Auth', (Auth) => Auth.logout()], - metaTags: { + tfMetaTags: { title: 'Logout' } }); diff --git a/modules/web-console/frontend/app/modules/states/password.state.js b/modules/web-console/frontend/app/modules/states/password.state.js index 587f83da9a1c5..8e326bfe9ab05 100644 --- a/modules/web-console/frontend/app/modules/states/password.state.js +++ b/modules/web-console/frontend/app/modules/states/password.state.js @@ -34,14 +34,14 @@ angular .state('password.reset', { url: '/reset?{token}', templateUrl, - metaTags: { + tfMetaTags: { title: 'Reset password' } }) .state('password.send', { url: '/send', templateUrl, - metaTags: { + tfMetaTags: { title: 'Password Send' } }); diff --git a/modules/web-console/frontend/app/modules/states/profile.state.js b/modules/web-console/frontend/app/modules/states/profile.state.js index 3298bdc0de6c7..87d1d022664b4 100644 --- a/modules/web-console/frontend/app/modules/states/profile.state.js +++ b/modules/web-console/frontend/app/modules/states/profile.state.js @@ -23,14 +23,13 @@ angular .module('ignite-console.states.profile', [ 'ui.router' ]) -.config(['$stateProvider', 'AclRouteProvider', function($stateProvider, AclRoute) { +.config(['$stateProvider', 'AclRouteProvider', function($stateProvider) { // set up the states - $stateProvider - .state('base.settings.profile', { + $stateProvider.state('base.settings.profile', { url: '/profile', templateUrl, - onEnter: AclRoute.checkAccess('profile'), - metaTags: { + permission: 'profile', + tfMetaTags: { title: 'User profile' } }); diff --git a/modules/web-console/frontend/app/modules/states/signin.state.js b/modules/web-console/frontend/app/modules/states/signin.state.js index b7be51d91b4b2..f5a496431dadc 100644 --- a/modules/web-console/frontend/app/modules/states/signin.state.js +++ b/modules/web-console/frontend/app/modules/states/signin.state.js @@ -24,30 +24,26 @@ angular // services 'ignite-console.user' ]) -.config(['$stateProvider', 'AclRouteProvider', function($stateProvider) { +.config(['$stateProvider', function($stateProvider) { // set up the states $stateProvider .state('signin', { url: '/', templateUrl, - resolve: { - user: ['$state', 'User', ($state, User) => { - return User.read() - .then(() => { - try { - const {name, params} = JSON.parse(localStorage.getItem('lastStateChangeSuccess')); + redirectTo: (trans) => { + return trans.injector().get('User').read() + .then(() => { + try { + const {name, params} = JSON.parse(localStorage.getItem('lastStateChangeSuccess')); - $state.go(name, params); - } catch (ignored) { - $state.go('base.configuration.tabs'); - } - }) - .catch(() => {}); - }] - }, - controllerAs: '$ctrl', - controller() {}, - metaTags: { + const restored = trans.router.stateService.target(name, params); + + return restored.valid() ? restored : 'base.configuration.tabs'; + } catch (ignored) { + return 'base.configuration.tabs'; + } + }) + .catch(() => true); } }); }]); diff --git a/modules/web-console/frontend/app/modules/user/AclRoute.provider.js b/modules/web-console/frontend/app/modules/user/AclRoute.provider.js index 4225bc4ed3721..f5556ffc8dd70 100644 --- a/modules/web-console/frontend/app/modules/user/AclRoute.provider.js +++ b/modules/web-console/frontend/app/modules/user/AclRoute.provider.js @@ -17,11 +17,9 @@ export default [() => { class AclRoute { - static checkAccess(permissions, failState) { - failState = failState || '403'; - + static checkAccess(permissions, failState = '403') { return ['$q', '$state', 'AclService', 'User', 'IgniteActivitiesData', function($q, $state, AclService, User, Activities) { - const action = this.name ? $state.href(this.name) : null; + const action = ''; return User.read() .catch(() => { diff --git a/modules/web-console/frontend/app/modules/user/user.module.js b/modules/web-console/frontend/app/modules/user/user.module.js index b86a62e492066..721d07fbe70c3 100644 --- a/modules/web-console/frontend/app/modules/user/user.module.js +++ b/modules/web-console/frontend/app/modules/user/user.module.js @@ -50,7 +50,7 @@ angular.module('ignite-console.user', [ .service(...Auth) .service(...User) .provider('AclRoute', AclRouteProvider) -.run(['$rootScope', 'AclService', ($root, AclService) => { +.run(['$rootScope', '$transitions', 'AclService', 'User', 'IgniteActivitiesData', ($root, $transitions, AclService, User, Activities) => { AclService.setAbilities(aclData); AclService.attachRole('guest'); @@ -70,4 +70,32 @@ angular.module('ignite-console.user', [ AclService.attachRole(role); }); + + $transitions.onBefore({}, (t) => { + const $state = t.router.stateService; + const {name, permission} = t.to(); + + return User.read() + .catch(() => { + User.clean(); + + if (name !== 'signin') + return $state.target('signin'); + + return true; + }) + .then(() => { + if (_.isEmpty(permission)) + return true; + + if (AclService.can(permission)) { + Activities.post({action: $state.href(name, t.params('to'))}); + + return true; + } + + return $state.target(t.to().failState || '403'); + }); + + }); }]); diff --git a/modules/web-console/frontend/app/services/CopyToClipboard.service.js b/modules/web-console/frontend/app/services/CopyToClipboard.service.js index 74c4764f60d7e..df0bb8a579ab6 100644 --- a/modules/web-console/frontend/app/services/CopyToClipboard.service.js +++ b/modules/web-console/frontend/app/services/CopyToClipboard.service.js @@ -38,10 +38,10 @@ export default ['IgniteCopyToClipboard', ['$window', 'IgniteMessages', ($window, if (document.execCommand('copy')) Messages.showInfo('Value copied to clipboard'); else - window.prompt('Copy to clipboard: Ctrl+C, Enter', toCopy); // eslint-disable-line no-alert + window.prompt('Copy to clipboard: Ctrl+C, Enter', toCopy); // eslint-disable-line no-alert } catch (err) { - window.prompt('Copy to clipboard: Ctrl+C, Enter', toCopy); // eslint-disable-line no-alert + window.prompt('Copy to clipboard: Ctrl+C, Enter', toCopy); // eslint-disable-line no-alert } textArea.remove(); diff --git a/modules/web-console/frontend/app/services/LegacyTable.service.js b/modules/web-console/frontend/app/services/LegacyTable.service.js index a024a3b7a858f..ab951647d6a52 100644 --- a/modules/web-console/frontend/app/services/LegacyTable.service.js +++ b/modules/web-console/frontend/app/services/LegacyTable.service.js @@ -25,7 +25,7 @@ export default ['IgniteLegacyTable', return item; path = path.replace(/\[(\w+)\]/g, '.$1'); // convert indexes to properties - path = path.replace(/^\./, ''); // strip a leading dot + path = path.replace(/^\./, ''); // strip a leading dot const segs = path.split('.'); let root = item; diff --git a/modules/web-console/frontend/app/vendor.js b/modules/web-console/frontend/app/vendor.js index 3bbb3224ce3b2..ea0f63c229413 100644 --- a/modules/web-console/frontend/app/vendor.js +++ b/modules/web-console/frontend/app/vendor.js @@ -24,9 +24,12 @@ import 'angular-strap'; import 'angular-strap/dist/angular-strap.tpl'; import 'angular-socket-io'; import 'angular-retina'; -import 'angular-ui-router'; + +import '@uirouter/angularjs'; +import '@uirouter/angularjs/lib/legacy/stateEvents'; + +import 'tf-metatags'; import 'angular-translate'; -import 'ui-router-metatags/dist/ui-router-metatags'; import 'angular-smart-table'; import 'angular-ui-grid/ui-grid'; import 'angular-drag-and-drop-lists'; @@ -49,6 +52,7 @@ import 'file-saver'; import 'jszip'; import 'nvd3'; import 'lodash'; + import 'angular-gridster/dist/angular-gridster.min.css'; import 'angular-tree-control/css/tree-control-attribute.css'; import 'angular-tree-control/css/tree-control.css'; diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json index 05c6eec2e198d..6b049ff91ae63 100644 --- a/modules/web-console/frontend/package.json +++ b/modules/web-console/frontend/package.json @@ -8,7 +8,7 @@ "dev": "npm start", "build": "webpack --config ./webpack/webpack.prod.babel.js", "test": "karma start ./test/karma.conf.js", - "eslint": "eslint --format node_modules/eslint-friendly-formatter gulpfile.babel.js/ app/ controllers/ ignite_modules/ -- --eff-by-issue" + "eslint": "eslint --format node_modules/eslint-friendly-formatter app/ controllers/ ignite_modules/ -- --eff-by-issue" }, "author": "", "contributors": [ @@ -45,13 +45,13 @@ "angular-socket-io": "0.7.0", "angular-strap": "2.3.12", "angular-touch": "1.5.11", - "angular-translate": "2.15.1", + "angular-translate": "2.15.2", "angular-tree-control": "0.2.28", - "angular-ui-grid": "4.0.4", - "angular-ui-router": "0.4.2", - "babel-core": "6.24.1", + "angular-ui-grid": "4.0.6", + "@uirouter/angularjs": "1.0.5", + "babel-core": "6.25.0", "babel-eslint": "7.2.3", - "babel-loader": "7.0.0", + "babel-loader": "7.1.1", "babel-plugin-add-module-exports": "0.2.1", "babel-plugin-transform-runtime": "6.23.0", "babel-polyfill": "6.23.0", @@ -61,45 +61,46 @@ "bootstrap-sass": "3.3.7", "brace": "0.10.0", "copy-webpack-plugin": "4.0.1", - "css-loader": "0.28.3", - "eslint": "3.19.0", + "css-loader": "0.28.4", + "eslint": "4.3.0", "eslint-friendly-formatter": "3.0.0", - "eslint-loader": "1.7.1", - "eslint-plugin-babel": "^4.1.1", + "eslint-loader": "1.9.0", + "eslint-plugin-babel": "4.1.1", "expose-loader": "0.7.3", - "extract-text-webpack-plugin": "2.1.0", - "file-loader": "0.11.1", + "extract-text-webpack-plugin": "3.0.0", + "file-loader": "0.11.2", "file-saver": "1.3.3", "font-awesome": "4.7.0", "glob": "7.1.2", "html-loader": "0.4.5", - "html-webpack-plugin": "2.28.0", + "html-webpack-plugin": "2.29.0", "jquery": "3.2.1", - "json-loader": "0.5.4", + "json-loader": "0.5.7", "jszip": "3.1.3", "lodash": "4.17.4", "node-sass": "4.5.3", "nvd3": "1.8.4", - "progress-bar-webpack-plugin": "1.9.3", + "progress-bar-webpack-plugin": "1.10.0", "pug-html-loader": "1.1.0", "pug-loader": "2.3.0", "raleway-webfont": "3.0.1", - "resolve-url-loader": "2.0.2", + "resolve-url-loader": "2.1.0", "roboto-font": "0.1.0", - "rxjs": "5.4.0", - "sass-loader": "6.0.5", + "rxjs": "5.4.2", + "sass-loader": "6.0.6", "socket.io-client": "1.7.3", - "style-loader": "0.18.1", - "svg-sprite-loader": "^3.0.5", - "ui-router-metatags": "1.0.3", - "webpack": "2.6.1", - "webpack-dev-server": "2.4.5", + "style-loader": "0.18.2", + "svg-sprite-loader": "3.0.7", + "tf-metatags": "2.0.0", + "webpack": "3.3.0", + "webpack-dev-server": "2.6.1", "webpack-merge": "4.1.0", - "worker-loader": "0.8.0" + "worker-loader": "0.8.1" }, "devDependencies": { - "chai": "4.0.2", - "jasmine-core": "2.6.2", + "chai": "4.1.0", + "type-detect": "4.0.3", + "jasmine-core": "2.6.4", "karma": "1.7.0", "karma-babel-preprocessor": "6.0.1", "karma-jasmine": "1.1.0", @@ -107,10 +108,10 @@ "karma-mocha-reporter": "2.2.3", "karma-phantomjs-launcher": "1.0.4", "karma-teamcity-reporter": "1.0.0", - "karma-webpack": "2.0.3", + "karma-webpack": "2.0.4", "mocha": "3.4.2", "mocha-teamcity-reporter": "1.1.1", "phantomjs-prebuilt": "2.1.14", - "sinon": "2.3.4" + "sinon": "2.3.8" } } diff --git a/modules/web-console/frontend/views/index.pug b/modules/web-console/frontend/views/index.pug index f8f13dd4dc7c0..1881b38b76bb8 100644 --- a/modules/web-console/frontend/views/index.pug +++ b/modules/web-console/frontend/views/index.pug @@ -23,23 +23,13 @@ html(ng-app='ignite-console' id='app' ng-strict-di) meta(http-equiv='content-language' content='en') meta(http-equiv='X-UA-Compatible' content='IE=Edge') - title(ng-bind='$meta.title') + title(ng-bind='tfMetaTags.title') + meta(ng-repeat='(key, value) in tfMetaTags.properties' name='{{::key}}' content='{{::value}}') meta(name='fragment' content='!') - meta(name='description' content='{{$meta.description}}') - meta(name='keywords' content='{{$meta.keywords}}') - meta(ng-repeat='(key, value) in $meta.properties' name='{{::key}}' content='{{::value}}') body.theme-line.body-overlap - .splash.splash-max-foreground(hide-on-state-change) - .splash-wrapper - .spinner - .bounce1 - .bounce2 - .bounce3 - - .splash-wellcome Loading... .ribbon-wrapper.right(ng-cloak) .ribbon(ng-style='IgniteDemoMode && {"background": "#1b6d88"}') From 2005eec081add6906fa204c4db0cfcc4a4907292 Mon Sep 17 00:00:00 2001 From: Andrey Novikov Date: Tue, 25 Jul 2017 15:49:21 +0700 Subject: [PATCH 021/547] IGNITE-5820 Web Console: Adjust modals height. --- .../web-console/frontend/app/primitives/modal/index.scss | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/modules/web-console/frontend/app/primitives/modal/index.scss b/modules/web-console/frontend/app/primitives/modal/index.scss index 87d8dc3b95083..fcf9885cce9e6 100644 --- a/modules/web-console/frontend/app/primitives/modal/index.scss +++ b/modules/web-console/frontend/app/primitives/modal/index.scss @@ -57,6 +57,15 @@ transform: translateZ(1px); } +.modal .modal-dialog--adjust-height { + margin-top: 0; + margin-bottom: 0; + + .modal-body { + max-height: calc(100vh - 150px); + } +} + .modal .modal-content { background-color: $gray-lighter; From bdc9e4b91403595c496a5fa4aca70228665feaaa Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 25 Jul 2017 15:15:43 +0300 Subject: [PATCH 022/547] IGNITE-5770 Refactor PlatformProcessor to PlatformTarget mechanism This closes #2336 --- .../platform/PlatformAbstractTarget.java | 6 +- .../processors/platform/PlatformIgnition.java | 5 +- .../platform/PlatformNoopProcessor.java | 132 ----- .../platform/PlatformProcessor.java | 216 ------- .../platform/PlatformProcessorImpl.java | 554 ++++++++++-------- .../cluster/PlatformClusterGroup.java | 29 + .../PlatformAtomicReference.java | 13 +- .../cpp/core/include/ignite/ignite.h | 6 +- .../ignite/impl/compute/compute_impl.h | 5 +- .../core/include/ignite/impl/ignite_impl.h | 112 ++-- .../ignite/impl/interop/interop_target.h | 16 +- modules/platforms/cpp/core/src/ignition.cpp | 23 +- .../cpp/core/src/impl/ignite_environment.cpp | 37 +- .../cpp/core/src/impl/ignite_impl.cpp | 25 +- .../core/src/impl/interop/interop_target.cpp | 18 +- .../cpp/jni/include/ignite/jni/exports.h | 31 +- .../cpp/jni/include/ignite/jni/java.h | 70 +-- .../platforms/cpp/jni/project/vs/module.def | 30 +- modules/platforms/cpp/jni/src/exports.cpp | 116 +--- modules/platforms/cpp/jni/src/java.cpp | 518 +--------------- .../dotnet/Apache.Ignite.Core/Ignition.cs | 12 +- .../Impl/Cache/CacheImpl.cs | 2 +- .../Impl/Cluster/ClusterGroupImpl.cs | 78 ++- .../Impl/Compute/ComputeImpl.cs | 6 +- .../Impl/Datastream/DataStreamerImpl.cs | 3 +- .../Impl/Datastream/StreamReceiverHolder.cs | 2 +- .../dotnet/Apache.Ignite.Core/Impl/Ignite.cs | 279 +++++---- .../Apache.Ignite.Core/Impl/Log/JavaLogger.cs | 23 +- .../Apache.Ignite.Core/Impl/PlatformTarget.cs | 18 +- .../Impl/Plugin/PluginContext.cs | 7 +- .../Impl/Unmanaged/IgniteJniNativeMethods.cs | 91 +-- .../Impl/Unmanaged/UnmanagedUtils.cs | 308 +--------- 32 files changed, 768 insertions(+), 2023 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformAbstractTarget.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformAbstractTarget.java index 5e785e22413e2..4a584f9724adf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformAbstractTarget.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformAbstractTarget.java @@ -130,7 +130,7 @@ public PlatformContext platformContext() { * @return Dummy value which is never returned. * @throws IgniteCheckedException Exception to be thrown. */ - private T throwUnsupported(int type) throws IgniteCheckedException { + public static T throwUnsupported(int type) throws IgniteCheckedException { throw new IgniteCheckedException("Unsupported operation type: " + type); } @@ -142,8 +142,8 @@ private T throwUnsupported(int type) throws IgniteCheckedException { * @param writer Writer. * @throws IgniteCheckedException In case of error. */ - protected PlatformListenable readAndListenFuture(BinaryRawReader reader, IgniteInternalFuture fut, - PlatformFutureUtils.Writer writer) + private PlatformListenable readAndListenFuture(BinaryRawReader reader, IgniteInternalFuture fut, + PlatformFutureUtils.Writer writer) throws IgniteCheckedException { long futId = reader.readLong(); int futTyp = reader.readInt(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformIgnition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformIgnition.java index 422e16e294c9d..754c69ed48ebd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformIgnition.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformIgnition.java @@ -49,9 +49,8 @@ public class PlatformIgnition { * @param factoryId Factory ID. * @param envPtr Environment pointer. * @param dataPtr Optional pointer to additional data required for startup. - * @return Ignite instance. */ - public static synchronized PlatformProcessor start(@Nullable String springCfgPath, + public static synchronized void start(@Nullable String springCfgPath, @Nullable String igniteInstanceName, int factoryId, long envPtr, long dataPtr) { if (envPtr <= 0) throw new IgniteException("Environment pointer must be positive."); @@ -78,8 +77,6 @@ public static synchronized PlatformProcessor start(@Nullable String springCfgPat PlatformProcessor old = instances.put(igniteInstanceName, proc); assert old == null; - - return proc; } finally { Thread.currentThread().setContextClassLoader(oldClsLdr); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformNoopProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformNoopProcessor.java index cd170ed24f5f9..f26826e8f9633 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformNoopProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformNoopProcessor.java @@ -23,7 +23,6 @@ import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.processors.GridProcessorAdapter; import org.apache.ignite.internal.processors.platform.cache.store.PlatformCacheStore; -import org.jetbrains.annotations.Nullable; /** * No-op processor. @@ -60,140 +59,9 @@ public PlatformNoopProcessor(GridKernalContext ctx) { // No-op. } - /** {@inheritDoc} */ - @Override public PlatformTargetProxy cache(@Nullable String name) throws IgniteCheckedException { - return null; - } - - /** {@inheritDoc} */ - @Override public PlatformTargetProxy createCache(@Nullable String name) throws IgniteCheckedException { - return null; - } - - /** {@inheritDoc} */ - @Override public PlatformTargetProxy getOrCreateCache(@Nullable String name) throws IgniteCheckedException { - return null; - } - - /** {@inheritDoc} */ - @Override public PlatformTargetProxy createCacheFromConfig(long memPtr) throws IgniteCheckedException { - return null; - } - - /** {@inheritDoc} */ - @Override public PlatformTargetProxy getOrCreateCacheFromConfig(long memPtr) throws IgniteCheckedException { - return null; - } - - /** {@inheritDoc} */ - @Override public void destroyCache(@Nullable String name) throws IgniteCheckedException { - // No-op. - } - - /** {@inheritDoc} */ - @Override public PlatformTargetProxy affinity(@Nullable String name) throws IgniteCheckedException { - return null; - } - - /** {@inheritDoc} */ - @Override public PlatformTargetProxy dataStreamer(@Nullable String cacheName, boolean keepBinary) - throws IgniteCheckedException { - return null; - } - - /** {@inheritDoc} */ - @Override public PlatformTargetProxy transactions() { - return null; - } - - /** {@inheritDoc} */ - @Override public PlatformTargetProxy projection() throws IgniteCheckedException { - return null; - } - - /** {@inheritDoc} */ - @Override public PlatformTargetProxy compute(PlatformTargetProxy grp) { - return null; - } - - /** {@inheritDoc} */ - @Override public PlatformTargetProxy message(PlatformTargetProxy grp) { - return null; - } - - /** {@inheritDoc} */ - @Override public PlatformTargetProxy events(PlatformTargetProxy grp) { - return null; - } - - /** {@inheritDoc} */ - @Override public PlatformTargetProxy services(PlatformTargetProxy grp) { - return null; - } - - /** {@inheritDoc} */ - @Override public PlatformTargetProxy extensions() { - return null; - } - - /** {@inheritDoc} */ - @Override public PlatformTargetProxy extension(int id) { - return null; - } - /** {@inheritDoc} */ @Override public void registerStore(PlatformCacheStore store, boolean convertBinary) throws IgniteCheckedException { // No-op. } - - /** {@inheritDoc} */ - @Override public PlatformTargetProxy atomicLong(String name, long initVal, boolean create) throws IgniteException { - return null; - } - - /** {@inheritDoc} */ - @Override public void getIgniteConfiguration(long memPtr) { - // No-op. - } - - /** {@inheritDoc} */ - @Override public void getCacheNames(long memPtr) { - // No-op. - } - - /** {@inheritDoc} */ - @Override public PlatformTargetProxy atomicSequence(String name, long initVal, boolean create) throws IgniteException { - return null; - } - - /** {@inheritDoc} */ - @Override public PlatformTargetProxy atomicReference(String name, long memPtr, boolean create) throws IgniteException { - return null; - } - - /** {@inheritDoc} */ - @Override public PlatformTargetProxy createNearCache(@Nullable String cacheName, long memPtr) { - return null; - } - - /** {@inheritDoc} */ - @Override public PlatformTargetProxy getOrCreateNearCache(@Nullable String cacheName, long memPtr) { - return null; - } - - /** {@inheritDoc} */ - @Override public boolean loggerIsLevelEnabled(int level) { - return false; - } - - /** {@inheritDoc} */ - @Override public void loggerLog(int level, String message, String category, String errorInfo) { - // No-op. - } - - /** {@inheritDoc} */ - @Override public PlatformTargetProxy binaryProcessor() { - return null; - } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformProcessor.java index 54f33a7c95703..9c17d788aa7da 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformProcessor.java @@ -61,139 +61,6 @@ public interface PlatformProcessor extends GridProcessor { */ public void awaitStart() throws IgniteCheckedException; - /** - * Get cache. - * - * @param name Cache name. - * @return Cache. - * @throws IgniteCheckedException If failed. - */ - public PlatformTargetProxy cache(@Nullable String name) throws IgniteCheckedException; - - /** - * Create cache. - * - * @param name Cache name. - * @return Cache. - * @throws IgniteCheckedException If failed. - */ - public PlatformTargetProxy createCache(@Nullable String name) throws IgniteCheckedException; - - /** - * Get or create cache. - * - * @param name Cache name. - * @return Cache. - * @throws IgniteCheckedException If failed. - */ - public PlatformTargetProxy getOrCreateCache(@Nullable String name) throws IgniteCheckedException; - - /** - * Create cache. - * - * @param memPtr Stream with cache config. - * @return Cache. - * @throws IgniteCheckedException If failed. - */ - public PlatformTargetProxy createCacheFromConfig(long memPtr) throws IgniteCheckedException; - - /** - * Get or create cache. - * - * @param memPtr Stream with cache config. - * @return Cache. - * @throws IgniteCheckedException If failed. - */ - public PlatformTargetProxy getOrCreateCacheFromConfig(long memPtr) throws IgniteCheckedException; - - /** - * Destroy dynamically created cache. - * - * @param name Cache name. - * @throws IgniteCheckedException If failed. - */ - public void destroyCache(@Nullable String name) throws IgniteCheckedException; - - /** - * Get affinity. - * - * @param name Cache name. - * @return Affinity. - * @throws IgniteCheckedException If failed. - */ - public PlatformTargetProxy affinity(@Nullable String name) throws IgniteCheckedException; - - /** - * Get data streamer. - * - * @param cacheName Cache name. - * @param keepBinary Binary flag. - * @return Data streamer. - * @throws IgniteCheckedException If failed. - */ - public PlatformTargetProxy dataStreamer(@Nullable String cacheName, boolean keepBinary) throws IgniteCheckedException; - - /** - * Get transactions. - * - * @return Transactions. - */ - public PlatformTargetProxy transactions(); - - /** - * Get projection. - * - * @return Projection. - * @throws IgniteCheckedException If failed. - */ - public PlatformTargetProxy projection() throws IgniteCheckedException; - - /** - * Create interop compute. - * - * @param grp Cluster group. - * @return Compute instance. - */ - public PlatformTargetProxy compute(PlatformTargetProxy grp); - - /** - * Create interop messaging. - * - * @param grp Cluster group. - * @return Messaging instance. - */ - public PlatformTargetProxy message(PlatformTargetProxy grp); - - /** - * Create interop events. - * - * @param grp Cluster group. - * @return Events instance. - */ - public PlatformTargetProxy events(PlatformTargetProxy grp); - - /** - * Create interop services. - * - * @param grp Cluster group. - * @return Services instance. - */ - public PlatformTargetProxy services(PlatformTargetProxy grp); - - /** - * Get platform extensions. Override this method to provide any additional targets and operations you need. - * - * @return Platform extensions. - */ - public PlatformTargetProxy extensions(); - - /** - * Gets platform extension by id. - * - * @return Platform extension target. - */ - public PlatformTargetProxy extension(int id); - /** * Register cache store. * @@ -202,87 +69,4 @@ public interface PlatformProcessor extends GridProcessor { * @throws IgniteCheckedException If failed. */ public void registerStore(PlatformCacheStore store, boolean convertBinary) throws IgniteCheckedException; - - /** - * Get or create AtomicLong. - * @param name Name. - * @param initVal Initial value. - * @param create Create flag. - * @return Platform atomic long. - */ - public PlatformTargetProxy atomicLong(String name, long initVal, boolean create); - - /** - * Get or create AtomicSequence. - * @param name Name. - * @param initVal Initial value. - * @param create Create flag. - * @return Platform atomic long. - */ - public PlatformTargetProxy atomicSequence(String name, long initVal, boolean create); - - /** - * Get or create AtomicReference. - * @param name Name. - * @param memPtr Pointer to a stream with initial value. 0 for null initial value. - * @param create Create flag. - * @return Platform atomic long. - */ - public PlatformTargetProxy atomicReference(String name, long memPtr, boolean create); - - /** - * Gets the configuration of the current Ignite instance. - * - * @param memPtr Stream to write data to. - */ - public void getIgniteConfiguration(long memPtr); - - /** - * Gets the cache names. - * - * @param memPtr Stream to write data to. - */ - public void getCacheNames(long memPtr); - - /** - * Starts a near cache on local node if cache was previously started. - * - * @param cacheName Cache name. - * @param memPtr Pointer to a stream with near cache config. 0 for default config. - * @return Cache. - */ - public PlatformTargetProxy createNearCache(@Nullable String cacheName, long memPtr); - - /** - * Gets existing near cache with the given name or creates a new one. - * - * @param cacheName Cache name. - * @param memPtr Pointer to a stream with near cache config. 0 for default config. - * @return Cache. - */ - public PlatformTargetProxy getOrCreateNearCache(@Nullable String cacheName, long memPtr); - - /** - * Gets a value indicating whether Ignite logger has specified level enabled. - * - * @param level Log level. - */ - public boolean loggerIsLevelEnabled(int level); - - /** - * Logs to the Ignite logger. - * - * @param level Log level. - * @param message Message. - * @param category Category. - * @param errorInfo Error info. - */ - public void loggerLog(int level, String message, String category, String errorInfo); - - /** - * Gets the binary processor. - * - * @return Binary processor. - */ - public PlatformTargetProxy binaryProcessor(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformProcessorImpl.java index 1da3112b3ae10..612f15447bb12 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformProcessorImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformProcessorImpl.java @@ -41,17 +41,13 @@ import org.apache.ignite.internal.processors.platform.cache.affinity.PlatformAffinity; import org.apache.ignite.internal.processors.platform.cache.store.PlatformCacheStore; import org.apache.ignite.internal.processors.platform.cluster.PlatformClusterGroup; -import org.apache.ignite.internal.processors.platform.compute.PlatformCompute; import org.apache.ignite.internal.processors.platform.datastreamer.PlatformDataStreamer; import org.apache.ignite.internal.processors.platform.datastructures.PlatformAtomicLong; import org.apache.ignite.internal.processors.platform.datastructures.PlatformAtomicReference; import org.apache.ignite.internal.processors.platform.datastructures.PlatformAtomicSequence; import org.apache.ignite.internal.processors.platform.dotnet.PlatformDotNetCacheStore; -import org.apache.ignite.internal.processors.platform.events.PlatformEvents; import org.apache.ignite.internal.processors.platform.memory.PlatformMemory; import org.apache.ignite.internal.processors.platform.memory.PlatformOutputStream; -import org.apache.ignite.internal.processors.platform.messaging.PlatformMessaging; -import org.apache.ignite.internal.processors.platform.services.PlatformServices; import org.apache.ignite.internal.processors.platform.transactions.PlatformTransactions; import org.apache.ignite.internal.processors.platform.utils.PlatformConfigurationUtils; import org.apache.ignite.internal.processors.platform.utils.PlatformUtils; @@ -74,7 +70,73 @@ * GridGain platform processor. */ @SuppressWarnings({"ConditionalExpressionWithIdenticalBranches", "unchecked"}) -public class PlatformProcessorImpl extends GridProcessorAdapter implements PlatformProcessor { +public class PlatformProcessorImpl extends GridProcessorAdapter implements PlatformProcessor, PlatformTarget { + /** */ + private static final int OP_GET_CACHE = 1; + + /** */ + private static final int OP_CREATE_CACHE = 2; + + /** */ + private static final int OP_GET_OR_CREATE_CACHE = 3; + + /** */ + private static final int OP_CREATE_CACHE_FROM_CONFIG = 4; + + /** */ + private static final int OP_GET_OR_CREATE_CACHE_FROM_CONFIG = 5; + + /** */ + private static final int OP_DESTROY_CACHE = 6; + + /** */ + private static final int OP_GET_AFFINITY = 7; + + /** */ + private static final int OP_GET_DATA_STREAMER = 8; + + /** */ + private static final int OP_GET_TRANSACTIONS = 9; + + /** */ + private static final int OP_GET_CLUSTER_GROUP = 10; + + /** */ + private static final int OP_GET_EXTENSION = 11; + + /** */ + private static final int OP_GET_ATOMIC_LONG = 12; + + /** */ + private static final int OP_GET_ATOMIC_REFERENCE = 13; + + /** */ + private static final int OP_GET_ATOMIC_SEQUENCE = 14; + + /** */ + private static final int OP_GET_IGNITE_CONFIGURATION = 15; + + /** */ + private static final int OP_GET_CACHE_NAMES = 16; + + /** */ + private static final int OP_CREATE_NEAR_CACHE = 17; + + /** */ + private static final int OP_GET_OR_CREATE_NEAR_CACHE = 18; + + /** */ + private static final int OP_LOGGER_IS_LEVEL_ENABLED = 19; + + /** */ + private static final int OP_LOGGER_LOG = 20; + + /** */ + private static final int OP_GET_BINARY_PROCESSOR = 21; + + /** */ + private static final int OP_RELEASE_START = 22; + /** Start latch. */ private final CountDownLatch startLatch = new CountDownLatch(1); @@ -157,7 +219,7 @@ public PlatformProcessorImpl(GridKernalContext ctx) { out.synchronize(); - platformCtx.gateway().onStart(this, mem.pointer()); + platformCtx.gateway().onStart(new PlatformTargetProxyImpl(this, platformCtx), mem.pointer()); } // At this moment all necessary native libraries must be loaded, so we can process with store creation. @@ -218,321 +280,360 @@ public PlatformProcessorImpl(GridKernalContext ctx) { } /** {@inheritDoc} */ - @Override public PlatformTargetProxy cache(@Nullable String name) throws IgniteCheckedException { - IgniteCacheProxy cache = (IgniteCacheProxy)ctx.grid().cache(name); + @Override public void registerStore(PlatformCacheStore store, boolean convertBinary) + throws IgniteCheckedException { + storeLock.readLock().lock(); - if (cache == null) - throw new IllegalArgumentException("Cache doesn't exist: " + name); + try { + if (stopped) + throw new IgniteCheckedException("Failed to initialize interop store because node is stopping: " + + store); - return createPlatformCache(cache); + if (started) + registerStore0(store, convertBinary); + else + pendingStores.add(new StoreInfo(store, convertBinary)); + } + finally { + storeLock.readLock().unlock(); + } } /** {@inheritDoc} */ - @Override public PlatformTargetProxy createCache(@Nullable String name) throws IgniteCheckedException { - IgniteCacheProxy cache = (IgniteCacheProxy)ctx.grid().createCache(name); - - assert cache != null; + @Override public void onDisconnected(IgniteFuture reconnectFut) throws IgniteCheckedException { + platformCtx.gateway().onClientDisconnected(); - return createPlatformCache(cache); + // 1) onReconnected is called on all grid components. + // 2) After all of grid components have completed their reconnection, reconnectFut is completed. + reconnectFut.listen(new CI1>() { + @Override public void apply(IgniteFuture future) { + platformCtx.gateway().onClientReconnected(clusterRestarted); + } + }); } /** {@inheritDoc} */ - @Override public PlatformTargetProxy getOrCreateCache(@Nullable String name) throws IgniteCheckedException { - IgniteCacheProxy cache = (IgniteCacheProxy)ctx.grid().getOrCreateCache(name); + @Override public IgniteInternalFuture onReconnected(boolean clusterRestarted) throws IgniteCheckedException { + // Save the flag value for callback of reconnectFut. + this.clusterRestarted = clusterRestarted; + + return null; + } + /** + * Creates new platform cache. + */ + private PlatformTarget createPlatformCache(IgniteCacheProxy cache) { assert cache != null; - return createPlatformCache(cache); + return new PlatformCache(platformCtx, cache, false, cacheExts); } - /** {@inheritDoc} */ - @Override public PlatformTargetProxy createCacheFromConfig(long memPtr) throws IgniteCheckedException { - BinaryRawReaderEx reader = platformCtx.reader(platformCtx.memory().get(memPtr)); - CacheConfiguration cfg = PlatformConfigurationUtils.readCacheConfiguration(reader); + /** + * Checks whether logger level is enabled. + * + * @param level Level. + * @return Result. + */ + private boolean loggerIsLevelEnabled(int level) { + IgniteLogger log = ctx.grid().log(); - IgniteCacheProxy cache = reader.readBoolean() - ? (IgniteCacheProxy)ctx.grid().createCache(cfg, PlatformConfigurationUtils.readNearConfiguration(reader)) - : (IgniteCacheProxy)ctx.grid().createCache(cfg); + switch (level) { + case PlatformLogger.LVL_TRACE: + return log.isTraceEnabled(); + case PlatformLogger.LVL_DEBUG: + return log.isDebugEnabled(); + case PlatformLogger.LVL_INFO: + return log.isInfoEnabled(); + case PlatformLogger.LVL_WARN: + return true; + case PlatformLogger.LVL_ERROR: + return true; + default: + assert false; + } - return createPlatformCache(cache); + return false; } - /** {@inheritDoc} */ - @Override public PlatformTargetProxy getOrCreateCacheFromConfig(long memPtr) throws IgniteCheckedException { - BinaryRawReaderEx reader = platformCtx.reader(platformCtx.memory().get(memPtr)); - CacheConfiguration cfg = PlatformConfigurationUtils.readCacheConfiguration(reader); + /** + * Logs to the Ignite logger. + * + * @param level Level. + * @param message Message. + * @param category Category. + * @param errorInfo Exception. + */ + private void loggerLog(int level, String message, String category, String errorInfo) { + IgniteLogger log = ctx.grid().log(); - IgniteCacheProxy cache = reader.readBoolean() - ? (IgniteCacheProxy)ctx.grid().getOrCreateCache(cfg, - PlatformConfigurationUtils.readNearConfiguration(reader)) - : (IgniteCacheProxy)ctx.grid().getOrCreateCache(cfg); + if (category != null) + log = log.getLogger(category); - return createPlatformCache(cache); - } + Throwable err = errorInfo == null ? null : new IgniteException("Platform error:" + errorInfo); - /** {@inheritDoc} */ - @Override public void destroyCache(@Nullable String name) throws IgniteCheckedException { - ctx.grid().destroyCache(name); + switch (level) { + case PlatformLogger.LVL_TRACE: + log.trace(message); + break; + case PlatformLogger.LVL_DEBUG: + log.debug(message); + break; + case PlatformLogger.LVL_INFO: + log.info(message); + break; + case PlatformLogger.LVL_WARN: + log.warning(message, err); + break; + case PlatformLogger.LVL_ERROR: + log.error(message, err); + break; + default: + assert false; + } } /** {@inheritDoc} */ - @Override public PlatformTargetProxy affinity(@Nullable String name) throws IgniteCheckedException { - return proxy(new PlatformAffinity(platformCtx, ctx, name)); + @Override public long processInLongOutLong(int type, long val) throws IgniteCheckedException { + switch (type) { + case OP_LOGGER_IS_LEVEL_ENABLED: { + return loggerIsLevelEnabled((int) val) ? PlatformAbstractTarget.TRUE : PlatformAbstractTarget.FALSE; + } + + case OP_RELEASE_START: { + releaseStart(); + + return 0; + } + } + + return PlatformAbstractTarget.throwUnsupported(type); } /** {@inheritDoc} */ - @Override public PlatformTargetProxy dataStreamer(@Nullable String cacheName, boolean keepBinary) - throws IgniteCheckedException { - IgniteDataStreamer ldr = ctx.dataStream().dataStreamer(cacheName); + @Override public long processInStreamOutLong(int type, BinaryRawReaderEx reader) throws IgniteCheckedException { + switch (type) { + case OP_DESTROY_CACHE: { + ctx.grid().destroyCache(reader.readString()); + + return 0; + } - ldr.keepBinary(true); + case OP_LOGGER_LOG: { + loggerLog(reader.readInt(), reader.readString(), reader.readString(), reader.readString()); - return proxy(new PlatformDataStreamer(platformCtx, cacheName, (DataStreamerImpl)ldr, keepBinary)); + return 0; + } + } + + return PlatformAbstractTarget.throwUnsupported(type); } /** {@inheritDoc} */ - @Override public PlatformTargetProxy transactions() { - return proxy(new PlatformTransactions(platformCtx)); + @Override public long processInStreamOutLong(int type, BinaryRawReaderEx reader, PlatformMemory mem) throws IgniteCheckedException { + return processInStreamOutLong(type, reader); } /** {@inheritDoc} */ - @Override public PlatformTargetProxy projection() throws IgniteCheckedException { - return proxy(new PlatformClusterGroup(platformCtx, ctx.grid().cluster())); + @Override public void processInStreamOutStream(int type, BinaryRawReaderEx reader, BinaryRawWriterEx writer) throws IgniteCheckedException { + PlatformAbstractTarget.throwUnsupported(type); } /** {@inheritDoc} */ - @Override public PlatformTargetProxy compute(PlatformTargetProxy grp) { - PlatformClusterGroup grp0 = (PlatformClusterGroup)grp.unwrap(); + @Override public PlatformTarget processInStreamOutObject(int type, BinaryRawReaderEx reader) throws IgniteCheckedException { + switch (type) { + case OP_GET_CACHE: { + String name = reader.readString(); - return proxy(new PlatformCompute(platformCtx, grp0.projection(), PlatformUtils.ATTR_PLATFORM)); - } + IgniteCacheProxy cache = (IgniteCacheProxy)ctx.grid().cache(name); - /** {@inheritDoc} */ - @Override public PlatformTargetProxy message(PlatformTargetProxy grp) { - PlatformClusterGroup grp0 = (PlatformClusterGroup)grp.unwrap(); + if (cache == null) + throw new IllegalArgumentException("Cache doesn't exist: " + name); - return proxy(new PlatformMessaging(platformCtx, grp0.projection().ignite().message(grp0.projection()))); - } + return createPlatformCache(cache); + } - /** {@inheritDoc} */ - @Override public PlatformTargetProxy events(PlatformTargetProxy grp) { - PlatformClusterGroup grp0 = (PlatformClusterGroup)grp.unwrap(); + case OP_CREATE_CACHE: { + String name = reader.readString(); - return proxy(new PlatformEvents(platformCtx, grp0.projection().ignite().events(grp0.projection()))); - } + IgniteCacheProxy cache = (IgniteCacheProxy)ctx.grid().createCache(name); - /** {@inheritDoc} */ - @Override public PlatformTargetProxy services(PlatformTargetProxy grp) { - PlatformClusterGroup grp0 = (PlatformClusterGroup)grp.unwrap(); + return createPlatformCache(cache); + } - return proxy(new PlatformServices(platformCtx, grp0.projection().ignite().services(grp0.projection()), false)); - } + case OP_GET_OR_CREATE_CACHE: { + String name = reader.readString(); - /** {@inheritDoc} */ - @Override public PlatformTargetProxy extensions() { - return null; - } + IgniteCacheProxy cache = (IgniteCacheProxy)ctx.grid().getOrCreateCache(name); - /** {@inheritDoc} */ - @Override public PlatformTargetProxy extension(int id) { - if (extensions != null && id < extensions.length) { - PlatformPluginExtension ext = extensions[id]; + return createPlatformCache(cache); + } - if (ext != null) - return proxy(ext.createTarget()); - } + case OP_CREATE_CACHE_FROM_CONFIG: { + CacheConfiguration cfg = PlatformConfigurationUtils.readCacheConfiguration(reader); - throw new IgniteException("Platform extension is not registered [id=" + id + ']'); - } + IgniteCacheProxy cache = reader.readBoolean() + ? (IgniteCacheProxy)ctx.grid().createCache(cfg, PlatformConfigurationUtils.readNearConfiguration(reader)) + : (IgniteCacheProxy)ctx.grid().createCache(cfg); - /** {@inheritDoc} */ - @Override public void registerStore(PlatformCacheStore store, boolean convertBinary) - throws IgniteCheckedException { - storeLock.readLock().lock(); + return createPlatformCache(cache); + } - try { - if (stopped) - throw new IgniteCheckedException("Failed to initialize interop store because node is stopping: " + - store); + case OP_GET_OR_CREATE_CACHE_FROM_CONFIG: { + CacheConfiguration cfg = PlatformConfigurationUtils.readCacheConfiguration(reader); - if (started) - registerStore0(store, convertBinary); - else - pendingStores.add(new StoreInfo(store, convertBinary)); - } - finally { - storeLock.readLock().unlock(); - } - } + IgniteCacheProxy cache = reader.readBoolean() + ? (IgniteCacheProxy)ctx.grid().getOrCreateCache(cfg, + PlatformConfigurationUtils.readNearConfiguration(reader)) + : (IgniteCacheProxy)ctx.grid().getOrCreateCache(cfg); - /** {@inheritDoc} */ - @Override public PlatformTargetProxy atomicLong(String name, long initVal, boolean create) throws IgniteException { - GridCacheAtomicLongImpl atomicLong = (GridCacheAtomicLongImpl)ignite().atomicLong(name, initVal, create); + return createPlatformCache(cache); + } - if (atomicLong == null) - return null; + case OP_GET_AFFINITY: { + return new PlatformAffinity(platformCtx, ctx, reader.readString()); + } - return proxy(new PlatformAtomicLong(platformCtx, atomicLong)); - } + case OP_GET_DATA_STREAMER: { + String cacheName = reader.readString(); + boolean keepBinary = reader.readBoolean(); - /** {@inheritDoc} */ - @Override public PlatformTargetProxy atomicSequence(String name, long initVal, boolean create) - throws IgniteException { - IgniteAtomicSequence atomicSeq = ignite().atomicSequence(name, initVal, create); + IgniteDataStreamer ldr = ctx.dataStream().dataStreamer(cacheName); - if (atomicSeq == null) - return null; + ldr.keepBinary(true); - return proxy(new PlatformAtomicSequence(platformCtx, atomicSeq)); - } + return new PlatformDataStreamer(platformCtx, cacheName, (DataStreamerImpl)ldr, keepBinary); + } - /** {@inheritDoc} */ - @Override public PlatformTargetProxy atomicReference(String name, long memPtr, boolean create) - throws IgniteException { - PlatformAtomicReference ref = PlatformAtomicReference.createInstance(platformCtx, name, memPtr, create); + case OP_GET_EXTENSION: { + int id = reader.readInt(); - return ref != null ? proxy(ref) : null; - } + if (extensions != null && id < extensions.length) { + PlatformPluginExtension ext = extensions[id]; - /** {@inheritDoc} */ - @Override public void onDisconnected(IgniteFuture reconnectFut) throws IgniteCheckedException { - platformCtx.gateway().onClientDisconnected(); + if (ext != null) { + return ext.createTarget(); + } + } - // 1) onReconnected is called on all grid components. - // 2) After all of grid components have completed their reconnection, reconnectFut is completed. - reconnectFut.listen(new CI1>() { - @Override public void apply(IgniteFuture future) { - platformCtx.gateway().onClientReconnected(clusterRestarted); + throw new IgniteException("Platform extension is not registered [id=" + id + ']'); } - }); - } - /** {@inheritDoc} */ - @Override public IgniteInternalFuture onReconnected(boolean clusterRestarted) throws IgniteCheckedException { - // Save the flag value for callback of reconnectFut. - this.clusterRestarted = clusterRestarted; + case OP_GET_ATOMIC_LONG: { + String name = reader.readString(); + long initVal = reader.readLong(); + boolean create = reader.readBoolean(); - return null; - } + GridCacheAtomicLongImpl atomicLong = (GridCacheAtomicLongImpl)ignite().atomicLong(name, initVal, create); - /** {@inheritDoc} */ - @Override public void getIgniteConfiguration(long memPtr) { - PlatformOutputStream stream = platformCtx.memory().get(memPtr).output(); - BinaryRawWriterEx writer = platformCtx.writer(stream); + if (atomicLong == null) + return null; - PlatformConfigurationUtils.writeIgniteConfiguration(writer, ignite().configuration()); + return new PlatformAtomicLong(platformCtx, atomicLong); + } - stream.synchronize(); - } + case OP_GET_ATOMIC_REFERENCE: { + String name = reader.readString(); + Object initVal = reader.readObjectDetached(); + boolean create = reader.readBoolean(); - /** {@inheritDoc} */ - @Override public void getCacheNames(long memPtr) { - PlatformOutputStream stream = platformCtx.memory().get(memPtr).output(); - BinaryRawWriterEx writer = platformCtx.writer(stream); + return PlatformAtomicReference.createInstance(platformCtx, name, initVal, create); + } - Collection names = ignite().cacheNames(); + case OP_GET_ATOMIC_SEQUENCE: { + String name = reader.readString(); + long initVal = reader.readLong(); + boolean create = reader.readBoolean(); - writer.writeInt(names.size()); + IgniteAtomicSequence atomicSeq = ignite().atomicSequence(name, initVal, create); - for (String name : names) - writer.writeString(name); + if (atomicSeq == null) + return null; - stream.synchronize(); - } + return new PlatformAtomicSequence(platformCtx, atomicSeq); + } - /** {@inheritDoc} */ - @Override public PlatformTargetProxy createNearCache(@Nullable String cacheName, long memPtr) { - NearCacheConfiguration cfg = getNearCacheConfiguration(memPtr); + case OP_CREATE_NEAR_CACHE: { + String cacheName = reader.readString(); - IgniteCacheProxy cache = (IgniteCacheProxy)ctx.grid().createNearCache(cacheName, cfg); + NearCacheConfiguration cfg = PlatformConfigurationUtils.readNearConfiguration(reader); - return createPlatformCache(cache); - } + IgniteCacheProxy cache = (IgniteCacheProxy)ctx.grid().createNearCache(cacheName, cfg); - /** {@inheritDoc} */ - @Override public PlatformTargetProxy getOrCreateNearCache(@Nullable String cacheName, long memPtr) { - NearCacheConfiguration cfg = getNearCacheConfiguration(memPtr); + return createPlatformCache(cache); + } + + case OP_GET_OR_CREATE_NEAR_CACHE: { + String cacheName = reader.readString(); - IgniteCacheProxy cache = (IgniteCacheProxy)ctx.grid().getOrCreateNearCache(cacheName, cfg); + NearCacheConfiguration cfg = PlatformConfigurationUtils.readNearConfiguration(reader); - return createPlatformCache(cache); + IgniteCacheProxy cache = (IgniteCacheProxy)ctx.grid().getOrCreateNearCache(cacheName, cfg); + + return createPlatformCache(cache); + } + } + + return PlatformAbstractTarget.throwUnsupported(type); } - /** - * Creates new platform cache. - */ - private PlatformTargetProxy createPlatformCache(IgniteCacheProxy cache) { - return proxy(new PlatformCache(platformCtx, cache, false, cacheExts)); + /** {@inheritDoc} */ + @Override public PlatformTarget processInObjectStreamOutObjectStream(int type, @Nullable PlatformTarget arg, + BinaryRawReaderEx reader, + BinaryRawWriterEx writer) + throws IgniteCheckedException { + return PlatformAbstractTarget.throwUnsupported(type); } /** {@inheritDoc} */ - @Override public boolean loggerIsLevelEnabled(int level) { - IgniteLogger log = ctx.grid().log(); + @Override public void processOutStream(int type, BinaryRawWriterEx writer) throws IgniteCheckedException { + switch (type) { + case OP_GET_IGNITE_CONFIGURATION: { + PlatformConfigurationUtils.writeIgniteConfiguration(writer, ignite().configuration()); - switch (level) { - case PlatformLogger.LVL_TRACE: - return log.isTraceEnabled(); - case PlatformLogger.LVL_DEBUG: - return log.isDebugEnabled(); - case PlatformLogger.LVL_INFO: - return log.isInfoEnabled(); - case PlatformLogger.LVL_WARN: - return true; - case PlatformLogger.LVL_ERROR: - return true; - default: - assert false; + return; + } + + case OP_GET_CACHE_NAMES: { + Collection names = ignite().cacheNames(); + + writer.writeInt(names.size()); + + for (String name : names) + writer.writeString(name); + + return; + } } - return false; + PlatformAbstractTarget.throwUnsupported(type); } /** {@inheritDoc} */ - @Override public void loggerLog(int level, String message, String category, String errorInfo) { - IgniteLogger log = ctx.grid().log(); + @Override public PlatformTarget processOutObject(int type) throws IgniteCheckedException { + switch (type) { + case OP_GET_TRANSACTIONS: + return new PlatformTransactions(platformCtx); - if (category != null) - log = log.getLogger(category); + case OP_GET_CLUSTER_GROUP: + return new PlatformClusterGroup(platformCtx, ctx.grid().cluster()); - Throwable err = errorInfo == null ? null : new IgniteException("Platform error:" + errorInfo); - - switch (level) { - case PlatformLogger.LVL_TRACE: - log.trace(message); - break; - case PlatformLogger.LVL_DEBUG: - log.debug(message); - break; - case PlatformLogger.LVL_INFO: - log.info(message); - break; - case PlatformLogger.LVL_WARN: - log.warning(message, err); - break; - case PlatformLogger.LVL_ERROR: - log.error(message, err); - break; - default: - assert false; + case OP_GET_BINARY_PROCESSOR: { + return new PlatformBinaryProcessor(platformCtx); + } } + + return PlatformAbstractTarget.throwUnsupported(type); } /** {@inheritDoc} */ - @Override public PlatformTargetProxy binaryProcessor() { - return proxy(new PlatformBinaryProcessor(platformCtx)); + @Override public PlatformAsyncResult processInStreamAsync(int type, BinaryRawReaderEx reader) throws IgniteCheckedException { + return PlatformAbstractTarget.throwUnsupported(type); } - /** - * Gets the near cache config. - * - * @param memPtr Memory pointer. - * @return Near config. - */ - private NearCacheConfiguration getNearCacheConfiguration(long memPtr) { - assert memPtr != 0; - - BinaryRawReaderEx reader = platformCtx.reader(platformCtx.memory().get(memPtr)); - return PlatformConfigurationUtils.readNearConfiguration(reader); + /** {@inheritDoc} */ + @Override public Exception convertException(Exception e) { + return e; } /** @@ -634,13 +735,6 @@ private static PlatformPluginExtension[] prepareExtensions(PlatformPluginExtensi return new PlatformPluginExtension[0]; } - /** - * Wraps target in a proxy. - */ - private PlatformTargetProxy proxy(PlatformTarget target) { - return new PlatformTargetProxyImpl(target, platformCtx); - } - /** * Store and manager pair. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cluster/PlatformClusterGroup.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cluster/PlatformClusterGroup.java index f6e3d2e4722bb..7c1c03e2214c4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cluster/PlatformClusterGroup.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cluster/PlatformClusterGroup.java @@ -36,6 +36,10 @@ import org.apache.ignite.internal.processors.platform.PlatformContext; import org.apache.ignite.internal.processors.platform.PlatformTarget; import org.apache.ignite.internal.processors.platform.cache.PlatformCache; +import org.apache.ignite.internal.processors.platform.compute.PlatformCompute; +import org.apache.ignite.internal.processors.platform.events.PlatformEvents; +import org.apache.ignite.internal.processors.platform.messaging.PlatformMessaging; +import org.apache.ignite.internal.processors.platform.services.PlatformServices; import org.apache.ignite.internal.processors.platform.utils.PlatformUtils; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.jetbrains.annotations.Nullable; @@ -126,6 +130,19 @@ public class PlatformClusterGroup extends PlatformAbstractTarget { /** */ private static final int OP_PERSISTENT_STORE_METRICS = 30; + /** */ + private static final int OP_GET_COMPUTE = 31; + + /** */ + private static final int OP_GET_MESSAGING = 32; + + /** */ + private static final int OP_GET_EVENTS = 33; + + /** */ + private static final int OP_GET_SERVICES = 34; + + /** Projection. */ private final ClusterGroupEx prj; @@ -381,6 +398,18 @@ public PlatformClusterGroup(PlatformContext platformCtx, ClusterGroupEx prj) { case OP_FOR_SERVERS: return new PlatformClusterGroup(platformCtx, (ClusterGroupEx)prj.forServers()); + + case OP_GET_COMPUTE: + return new PlatformCompute(platformCtx, prj, PlatformUtils.ATTR_PLATFORM); + + case OP_GET_MESSAGING: + return new PlatformMessaging(platformCtx, platformCtx.kernalContext().grid().message(prj)); + + case OP_GET_EVENTS: + return new PlatformEvents(platformCtx, platformCtx.kernalContext().grid().events(prj)); + + case OP_GET_SERVICES: + return new PlatformServices(platformCtx, platformCtx.kernalContext().grid().services(prj),false); } return super.processOutObject(type); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/datastructures/PlatformAtomicReference.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/datastructures/PlatformAtomicReference.java index a6442590d7b46..93c00408f458e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/datastructures/PlatformAtomicReference.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/datastructures/PlatformAtomicReference.java @@ -23,7 +23,6 @@ import org.apache.ignite.internal.processors.datastructures.GridCacheAtomicReferenceImpl; import org.apache.ignite.internal.processors.platform.PlatformAbstractTarget; import org.apache.ignite.internal.processors.platform.PlatformContext; -import org.apache.ignite.internal.processors.platform.memory.PlatformMemory; /** * Platform atomic reference wrapper. @@ -53,23 +52,15 @@ public class PlatformAtomicReference extends PlatformAbstractTarget { * * @param ctx Context. * @param name Name. - * @param memPtr Pointer to a stream with initial value. 0 for default value. + * @param initVal Initial value. * @param create Create flag. * @return Instance of a PlatformAtomicReference, or null when Ignite reference with specific name is null. */ - public static PlatformAtomicReference createInstance(PlatformContext ctx, String name, long memPtr, + public static PlatformAtomicReference createInstance(PlatformContext ctx, String name, Object initVal, boolean create) { assert ctx != null; assert name != null; - Object initVal = null; - - if (memPtr != 0) { - try (PlatformMemory mem = ctx.memory().get(memPtr)) { - initVal = ctx.reader(mem).readObjectDetached(); - } - } - GridCacheAtomicReferenceImpl atomicRef = (GridCacheAtomicReferenceImpl)ctx.kernalContext().grid().atomicReference(name, initVal, create); diff --git a/modules/platforms/cpp/core/include/ignite/ignite.h b/modules/platforms/cpp/core/include/ignite/ignite.h index b3b06f034d57f..07134a1a55577 100644 --- a/modules/platforms/cpp/core/include/ignite/ignite.h +++ b/modules/platforms/cpp/core/include/ignite/ignite.h @@ -102,7 +102,7 @@ namespace ignite template cache::Cache GetCache(const char* name, IgniteError& err) { - impl::cache::CacheImpl* cacheImpl = impl.Get()->GetCache(name, err); + impl::cache::CacheImpl* cacheImpl = impl.Get()->GetCache(name, err); return cache::Cache(cacheImpl); } @@ -139,7 +139,7 @@ namespace ignite template cache::Cache GetOrCreateCache(const char* name, IgniteError& err) { - impl::cache::CacheImpl* cacheImpl = impl.Get()->GetOrCreateCache(name, err); + impl::cache::CacheImpl* cacheImpl = impl.Get()->GetOrCreateCache(name, err); return cache::Cache(cacheImpl); } @@ -176,7 +176,7 @@ namespace ignite template cache::Cache CreateCache(const char* name, IgniteError& err) { - impl::cache::CacheImpl* cacheImpl = impl.Get()->CreateCache(name, err); + impl::cache::CacheImpl* cacheImpl = impl.Get()->CreateCache(name, err); return cache::Cache(cacheImpl); } diff --git a/modules/platforms/cpp/core/include/ignite/impl/compute/compute_impl.h b/modules/platforms/cpp/core/include/ignite/impl/compute/compute_impl.h index 4ba1c1cb9b61c..2b04dcbc1bc53 100644 --- a/modules/platforms/cpp/core/include/ignite/impl/compute/compute_impl.h +++ b/modules/platforms/cpp/core/include/ignite/impl/compute/compute_impl.h @@ -208,7 +208,10 @@ namespace ignite out.Synchronize(); - jobject target = InStreamOutObject(operation, *mem.Get()); + IgniteError err; + jobject target = InStreamOutObject(operation, *mem.Get(), err); + IgniteError::ThrowIfNeeded(err); + std::auto_ptr cancelable(new CancelableImpl(GetEnvironmentPointer(), target)); return cancelable; diff --git a/modules/platforms/cpp/core/include/ignite/impl/ignite_impl.h b/modules/platforms/cpp/core/include/ignite/impl/ignite_impl.h index baddec49b1038..d1763c4316a58 100644 --- a/modules/platforms/cpp/core/include/ignite/impl/ignite_impl.h +++ b/modules/platforms/cpp/core/include/ignite/impl/ignite_impl.h @@ -28,14 +28,34 @@ #include #include +using namespace ignite::impl::interop; +using namespace ignite::common::concurrent; +using namespace ignite::impl::binary; +using namespace ignite::binary; + namespace ignite { namespace impl { + /* + * PlatformProcessor op codes. + */ + struct ProcessorOp + { + enum Type + { + GET_CACHE = 1, + CREATE_CACHE = 2, + GET_OR_CREATE_CACHE = 3, + GET_TRANSACTIONS = 9, + GET_CLUSTER_GROUP = 10, + }; + }; + /** * Ignite implementation. */ - class IGNITE_FRIEND_EXPORT IgniteImpl + class IGNITE_FRIEND_EXPORT IgniteImpl : private interop::InteropTarget { typedef common::concurrent::SharedPointer SP_IgniteEnvironment; typedef common::concurrent::SharedPointer SP_TransactionsImpl; @@ -48,13 +68,8 @@ namespace ignite * @param env Environment. * @param javaRef Reference to java object. */ - IgniteImpl(SP_IgniteEnvironment env, jobject javaRef); + IgniteImpl(SP_IgniteEnvironment env); - /** - * Destructor. - */ - ~IgniteImpl(); - /** * Get name of the Ignite. * @@ -82,23 +97,9 @@ namespace ignite * @param name Cache name. * @param err Error. */ - template cache::CacheImpl* GetCache(const char* name, IgniteError& err) { - ignite::jni::java::JniErrorInfo jniErr; - - jobject cacheJavaRef = env.Get()->Context()->ProcessorCache(javaRef, name, &jniErr); - - if (!cacheJavaRef) - { - IgniteError::SetError(jniErr.code, jniErr.errCls, jniErr.errMsg, err); - - return NULL; - } - - char* name0 = common::CopyChars(name); - - return new cache::CacheImpl(name0, env, cacheJavaRef); + return GetOrCreateCache(name, err, ProcessorOp::GET_CACHE); } /** @@ -107,23 +108,9 @@ namespace ignite * @param name Cache name. * @param err Error. */ - template cache::CacheImpl* GetOrCreateCache(const char* name, IgniteError& err) { - ignite::jni::java::JniErrorInfo jniErr; - - jobject cacheJavaRef = env.Get()->Context()->ProcessorGetOrCreateCache(javaRef, name, &jniErr); - - if (!cacheJavaRef) - { - IgniteError::SetError(jniErr.code, jniErr.errCls, jniErr.errMsg, err); - - return NULL; - } - - char* name0 = common::CopyChars(name); - - return new cache::CacheImpl(name0, env, cacheJavaRef); + return GetOrCreateCache(name, err, ProcessorOp::GET_OR_CREATE_CACHE); } /** @@ -132,23 +119,9 @@ namespace ignite * @param name Cache name. * @param err Error. */ - template cache::CacheImpl* CreateCache(const char* name, IgniteError& err) { - ignite::jni::java::JniErrorInfo jniErr; - - jobject cacheJavaRef = env.Get()->Context()->ProcessorCreateCache(javaRef, name, &jniErr); - - if (!cacheJavaRef) - { - IgniteError::SetError(jniErr.code, jniErr.errCls, jniErr.errMsg, err); - - return NULL; - } - - char* name0 = common::CopyChars(name); - - return new cache::CacheImpl(name0, env, cacheJavaRef); + return GetOrCreateCache(name, err, ProcessorOp::CREATE_CACHE); } /** @@ -227,9 +200,6 @@ namespace ignite /** Environment. */ SP_IgniteEnvironment env; - /** Native Java counterpart. */ - jobject javaRef; - /** Transactions implementaion. */ SP_TransactionsImpl txImpl; @@ -237,6 +207,38 @@ namespace ignite cluster::SP_ClusterGroupImpl prjImpl; IGNITE_NO_COPY_ASSIGNMENT(IgniteImpl) + + /** + * Get or create cache. + * + * @param name Cache name. + * @param err Error. + * @param op Operation code. + */ + cache::CacheImpl* GetOrCreateCache(const char* name, IgniteError& err, int32_t op) + { + SharedPointer mem = env.Get()->AllocateMemory(); + InteropMemory* mem0 = mem.Get(); + InteropOutputStream out(mem0); + BinaryWriterImpl writer(&out, env.Get()->GetTypeManager()); + BinaryRawWriter rawWriter(&writer); + + rawWriter.WriteString(name); + + out.Synchronize(); + + jobject cacheJavaRef = InStreamOutObject(op, *mem0, err); + + if (!cacheJavaRef) + { + return NULL; + } + + char* name0 = common::CopyChars(name); + + return new cache::CacheImpl(name0, env, cacheJavaRef); + } + }; } } diff --git a/modules/platforms/cpp/core/include/ignite/impl/interop/interop_target.h b/modules/platforms/cpp/core/include/ignite/impl/interop/interop_target.h index 0384dcc1ed7b5..29a3a2da790d0 100644 --- a/modules/platforms/cpp/core/include/ignite/impl/interop/interop_target.h +++ b/modules/platforms/cpp/core/include/ignite/impl/interop/interop_target.h @@ -59,6 +59,16 @@ namespace ignite */ InteropTarget(ignite::common::concurrent::SharedPointer env, jobject javaRef); + /** + * Constructor used to create new instance. + * + * @param env Environment. + * @param javaRef Reference to java object. + * @param javaRef Whether javaRef release in destructor should be skipped. + */ + InteropTarget(ignite::common::concurrent::SharedPointer env, jobject javaRef, + bool skipJavaRefRelease); + /** * Destructor. */ @@ -139,9 +149,10 @@ namespace ignite * * @param opType Type of operation. * @param outInMem Input and output memory. + * @param err Error. * @return Java object references. */ - jobject InStreamOutObject(int32_t opType, InteropMemory& outInMem); + jobject InStreamOutObject(int32_t opType, InteropMemory& outInMem, IgniteError& err); /** * Internal out-in operation. @@ -190,6 +201,9 @@ namespace ignite /** Handle to Java object. */ jobject javaRef; + /** javaRef release flag. */ + bool skipJavaRefRelease; + IGNITE_NO_COPY_ASSIGNMENT(InteropTarget) /** diff --git a/modules/platforms/cpp/core/src/ignition.cpp b/modules/platforms/cpp/core/src/ignition.cpp index e12a2085b6d40..bc25b07adcfb5 100644 --- a/modules/platforms/cpp/core/src/ignition.cpp +++ b/modules/platforms/cpp/core/src/ignition.cpp @@ -301,13 +301,13 @@ namespace ignite stream.WriteBool(false); stream.Synchronize(); - jobject javaRef = ctx.Get()->IgnitionStart(&springCfgPath0[0], namep, 2, mem.PointerLong(), &jniErr); + ctx.Get()->IgnitionStart(&springCfgPath0[0], namep, 2, mem.PointerLong(), &jniErr); // Releasing control over environment as it is controlled by Java at this point. // Even if the call has failed environment are going to be released by the Java. envTarget.release(); - if (!javaRef) + if (!env.Get()->GetProcessor()) { IgniteError::SetError(jniErr.code, jniErr.errCls, jniErr.errMsg, err); @@ -323,7 +323,7 @@ namespace ignite env.Get()->ProcessorReleaseStart(); - IgniteImpl* impl = new IgniteImpl(env, javaRef); + IgniteImpl* impl = new IgniteImpl(env); return Ignite(impl); } @@ -383,22 +383,9 @@ namespace ignite SharedPointer* env = static_cast*>(hnds->target); - // 4. Get fresh node reference. - jobject ref = ctx.Get()->IgnitionInstance(name0, &jniErr); + IgniteImpl* impl = new IgniteImpl(*env); - if (err.GetCode() == IgniteError::IGNITE_SUCCESS) { - if (ref) - { - IgniteImpl* impl = new IgniteImpl(*env, ref); - - res = Ignite(impl); - } - else - // Error: concurrent node stop. - err = IgniteError(IgniteError::IGNITE_ERR_GENERIC, - "Failed to get Ignite instance because it was stopped concurrently."); - - } + res = Ignite(impl); } else // Error: no node with the given name. diff --git a/modules/platforms/cpp/core/src/impl/ignite_environment.cpp b/modules/platforms/cpp/core/src/impl/ignite_environment.cpp index 4e78f095f4822..e5ce0043555f2 100644 --- a/modules/platforms/cpp/core/src/impl/ignite_environment.cpp +++ b/modules/platforms/cpp/core/src/impl/ignite_environment.cpp @@ -61,7 +61,30 @@ namespace ignite ON_START = 49, ON_STOP = 50, COMPUTE_TASK_LOCAL_JOB_RESULT = 60, - COMPUTE_JOB_EXECUTE_LOCAL = 61 + COMPUTE_JOB_EXECUTE_LOCAL = 61, + }; + }; + + /* + * PlatformProcessor op codes. + */ + struct ProcessorOp + { + enum Type + { + GET_BINARY_PROCESSOR = 21, + RELEASE_START = 22 + }; + }; + + /* + * PlatformClusterGroup op codes. + */ + struct ClusterGroupOp + { + enum Type + { + GET_COMPUTE = 31 }; }; @@ -298,7 +321,10 @@ namespace ignite { latch.CountDown(); - jobject binaryProc = Context()->ProcessorBinaryProcessor(proc.Get()); + JniErrorInfo jniErr; + + jobject binaryProc = Context()->TargetOutObject(proc.Get(), ProcessorOp::GET_BINARY_PROCESSOR, &jniErr); + metaUpdater = new BinaryTypeUpdaterImpl(*this, binaryProc); metaMgr->SetUpdater(metaUpdater); @@ -375,7 +401,7 @@ namespace ignite { JniErrorInfo jniErr; - jobject res = ctx.Get()->ProcessorCompute(proc.Get(), proj, &jniErr); + jobject res = ctx.Get()->TargetOutObject(proj, ClusterGroupOp::GET_COMPUTE, &jniErr); IgniteError err; @@ -557,7 +583,10 @@ namespace ignite void IgniteEnvironment::ProcessorReleaseStart() { if (proc.Get()) - ctx.Get()->ProcessorReleaseStart(proc.Get()); + { + JniErrorInfo jniErr; + ctx.Get()->TargetInLongOutLong(proc.Get(), ProcessorOp::RELEASE_START, 0, &jniErr); + } } HandleRegistry& IgniteEnvironment::GetHandleRegistry() diff --git a/modules/platforms/cpp/core/src/impl/ignite_impl.cpp b/modules/platforms/cpp/core/src/impl/ignite_impl.cpp index 16e954c6ceac9..f7ff1859e05a5 100644 --- a/modules/platforms/cpp/core/src/impl/ignite_impl.cpp +++ b/modules/platforms/cpp/core/src/impl/ignite_impl.cpp @@ -24,9 +24,9 @@ namespace ignite { namespace impl { - IgniteImpl::IgniteImpl(SharedPointer env, jobject javaRef) : - env(env), - javaRef(javaRef) + IgniteImpl::IgniteImpl(SharedPointer env) : + InteropTarget(env, static_cast(env.Get()->GetProcessor()), true), + env(env) { IgniteError err; @@ -39,11 +39,6 @@ namespace ignite IgniteError::ThrowIfNeeded(err); } - IgniteImpl::~IgniteImpl() - { - JniContext::Release(javaRef); - } - const char* IgniteImpl::GetName() const { return env.Get()->InstanceName(); @@ -75,14 +70,10 @@ namespace ignite { SP_TransactionsImpl res; - JniErrorInfo jniErr; - - jobject txJavaRef = env.Get()->Context()->ProcessorTransactions(javaRef, &jniErr); + jobject txJavaRef = InOpObject(ProcessorOp::GET_TRANSACTIONS, err); if (txJavaRef) res = SP_TransactionsImpl(new transactions::TransactionsImpl(env, txJavaRef)); - else - IgniteError::SetError(jniErr.code, jniErr.errCls, jniErr.errMsg, err); return res; } @@ -93,12 +84,10 @@ namespace ignite JniErrorInfo jniErr; - jobject txJavaRef = env.Get()->Context()->ProcessorProjection(javaRef, &jniErr); + jobject clusterGroupJavaRef = InOpObject(ProcessorOp::GET_CLUSTER_GROUP, err); - if (txJavaRef) - res = cluster::SP_ClusterGroupImpl(new cluster::ClusterGroupImpl(env, txJavaRef)); - else - IgniteError::SetError(jniErr.code, jniErr.errCls, jniErr.errMsg, err); + if (clusterGroupJavaRef) + res = cluster::SP_ClusterGroupImpl(new cluster::ClusterGroupImpl(env, clusterGroupJavaRef)); return res; } diff --git a/modules/platforms/cpp/core/src/impl/interop/interop_target.cpp b/modules/platforms/cpp/core/src/impl/interop/interop_target.cpp index 7eed6f3d507df..70db2c98a059b 100644 --- a/modules/platforms/cpp/core/src/impl/interop/interop_target.cpp +++ b/modules/platforms/cpp/core/src/impl/interop/interop_target.cpp @@ -32,14 +32,24 @@ namespace ignite namespace interop { InteropTarget::InteropTarget(SharedPointer env, jobject javaRef) : - env(env), javaRef(javaRef) + env(env), javaRef(javaRef), skipJavaRefRelease(false) + { + // No-op. + } + + InteropTarget::InteropTarget(SharedPointer env, jobject javaRef, + bool skipJavaRefRelease) : + env(env), javaRef(javaRef), skipJavaRefRelease(skipJavaRefRelease) { // No-op. } InteropTarget::~InteropTarget() { - JniContext::Release(javaRef); + if (!skipJavaRefRelease) + { + JniContext::Release(javaRef); + } } int64_t InteropTarget::WriteTo(InteropMemory* mem, InputOperation& inOp, IgniteError& err) @@ -216,7 +226,7 @@ namespace ignite return OperationResult::AI_ERROR; } - jobject InteropTarget::InStreamOutObject(int32_t opType, InteropMemory& outInMem) + jobject InteropTarget::InStreamOutObject(int32_t opType, InteropMemory& outInMem, IgniteError& err) { JniErrorInfo jniErr; @@ -226,9 +236,7 @@ namespace ignite { jobject res = env.Get()->Context()->TargetInStreamOutObject(javaRef, opType, outInPtr, &jniErr); - IgniteError err; IgniteError::SetError(jniErr.code, jniErr.errCls, jniErr.errMsg, err); - IgniteError::ThrowIfNeeded(err); return res; } diff --git a/modules/platforms/cpp/jni/include/ignite/jni/exports.h b/modules/platforms/cpp/jni/include/ignite/jni/exports.h index 6fe91c7d28d47..ea0c32a072ad7 100644 --- a/modules/platforms/cpp/jni/include/ignite/jni/exports.h +++ b/modules/platforms/cpp/jni/include/ignite/jni/exports.h @@ -25,40 +25,11 @@ namespace gcj = ignite::jni::java; extern "C" { int IGNITE_CALL IgniteReallocate(long long memPtr, int cap); - void* IGNITE_CALL IgniteIgnitionStart(gcj::JniContext* ctx, char* cfgPath, char* name, int factoryId, long long dataPtr); - void* IGNITE_CALL IgniteIgnitionInstance(gcj::JniContext* ctx, char* name); + void IGNITE_CALL IgniteIgnitionStart(gcj::JniContext* ctx, char* cfgPath, char* name, int factoryId, long long dataPtr); long long IGNITE_CALL IgniteIgnitionEnvironmentPointer(gcj::JniContext* ctx, char* name); bool IGNITE_CALL IgniteIgnitionStop(gcj::JniContext* ctx, char* name, bool cancel); void IGNITE_CALL IgniteIgnitionStopAll(gcj::JniContext* ctx, bool cancel); - void IGNITE_CALL IgniteProcessorReleaseStart(gcj::JniContext* ctx, void* obj); - void* IGNITE_CALL IgniteProcessorProjection(gcj::JniContext* ctx, void* obj); - void* IGNITE_CALL IgniteProcessorCache(gcj::JniContext* ctx, void* obj, char* name); - void* IGNITE_CALL IgniteProcessorCreateCache(gcj::JniContext* ctx, void* obj, char* name); - void* IGNITE_CALL IgniteProcessorGetOrCreateCache(gcj::JniContext* ctx, void* obj, char* name); - void* IGNITE_CALL IgniteProcessorCreateCacheFromConfig(gcj::JniContext* ctx, void* obj, long long memPtr); - void* IGNITE_CALL IgniteProcessorGetOrCreateCacheFromConfig(gcj::JniContext* ctx, void* obj, long long memPtr); - void* IGNITE_CALL IgniteProcessorCreateNearCache(gcj::JniContext* ctx, void* obj, char* name, long long memPtr); - void* IGNITE_CALL IgniteProcessorGetOrCreateNearCache(gcj::JniContext* ctx, void* obj, char* name, long long memPtr); - void IGNITE_CALL IgniteProcessorDestroyCache(gcj::JniContext* ctx, void* obj, char* name); - void* IGNITE_CALL IgniteProcessorAffinity(gcj::JniContext* ctx, void* obj, char* name); - void* IGNITE_CALL IgniteProcessorDataStreamer(gcj::JniContext* ctx, void* obj, char* name, bool keepPortable); - void* IGNITE_CALL IgniteProcessorTransactions(gcj::JniContext* ctx, void* obj); - void* IGNITE_CALL IgniteProcessorCompute(gcj::JniContext* ctx, void* obj, void* prj); - void* IGNITE_CALL IgniteProcessorMessage(gcj::JniContext* ctx, void* obj, void* prj); - void* IGNITE_CALL IgniteProcessorEvents(gcj::JniContext* ctx, void* obj, void* prj); - void* IGNITE_CALL IgniteProcessorServices(gcj::JniContext* ctx, void* obj, void* prj); - void* IGNITE_CALL IgniteProcessorExtensions(gcj::JniContext* ctx, void* obj); - void* IGNITE_CALL IgniteProcessorExtension(gcj::JniContext* ctx, void* obj, int id); - void* IGNITE_CALL IgniteProcessorAtomicLong(gcj::JniContext* ctx, void* obj, char* name, long long initVal, bool create); - void* IGNITE_CALL IgniteProcessorAtomicSequence(gcj::JniContext* ctx, void* obj, char* name, long long initVal, bool create); - void* IGNITE_CALL IgniteProcessorAtomicReference(gcj::JniContext* ctx, void* obj, char* name, long long memPtr, bool create); - void IGNITE_CALL IgniteProcessorGetIgniteConfiguration(gcj::JniContext* ctx, void* obj, long long memPtr); - void IGNITE_CALL IgniteProcessorGetCacheNames(gcj::JniContext* ctx, void* obj, long long memPtr); - bool IGNITE_CALL IgniteProcessorLoggerIsLevelEnabled(gcj::JniContext* ctx, void* obj, int level); - void IGNITE_CALL IgniteProcessorLoggerLog(gcj::JniContext* ctx, void* obj, int level, char* message, char* category, char* errorInfo); - void* IGNITE_CALL IgniteProcessorBinaryProcessor(gcj::JniContext* ctx, void* obj); - long long IGNITE_CALL IgniteTargetInLongOutLong(gcj::JniContext* ctx, void* obj, int opType, long long memPtr); long long IGNITE_CALL IgniteTargetInStreamOutLong(gcj::JniContext* ctx, void* obj, int opType, long long memPtr); void IGNITE_CALL IgniteTargetInStreamOutStream(gcj::JniContext* ctx, void* obj, int opType, long long inMemPtr, long long outMemPtr); diff --git a/modules/platforms/cpp/jni/include/ignite/jni/java.h b/modules/platforms/cpp/jni/include/ignite/jni/java.h index f6d72077e0041..c170a5bb56755 100644 --- a/modules/platforms/cpp/jni/include/ignite/jni/java.h +++ b/modules/platforms/cpp/jni/include/ignite/jni/java.h @@ -175,32 +175,6 @@ namespace ignite jclass c_PlatformProcessor; jmethodID m_PlatformProcessor_releaseStart; - jmethodID m_PlatformProcessor_cache; - jmethodID m_PlatformProcessor_createCache; - jmethodID m_PlatformProcessor_getOrCreateCache; - jmethodID m_PlatformProcessor_createCacheFromConfig; - jmethodID m_PlatformProcessor_getOrCreateCacheFromConfig; - jmethodID m_PlatformProcessor_createNearCache; - jmethodID m_PlatformProcessor_getOrCreateNearCache; - jmethodID m_PlatformProcessor_destroyCache; - jmethodID m_PlatformProcessor_affinity; - jmethodID m_PlatformProcessor_dataStreamer; - jmethodID m_PlatformProcessor_transactions; - jmethodID m_PlatformProcessor_projection; - jmethodID m_PlatformProcessor_compute; - jmethodID m_PlatformProcessor_message; - jmethodID m_PlatformProcessor_events; - jmethodID m_PlatformProcessor_services; - jmethodID m_PlatformProcessor_extensions; - jmethodID m_PlatformProcessor_extension; - jmethodID m_PlatformProcessor_atomicLong; - jmethodID m_PlatformProcessor_getIgniteConfiguration; - jmethodID m_PlatformProcessor_getCacheNames; - jmethodID m_PlatformProcessor_atomicSequence; - jmethodID m_PlatformProcessor_atomicReference; - jmethodID m_PlatformProcessor_loggerIsLevelEnabled; - jmethodID m_PlatformProcessor_loggerLog; - jmethodID m_PlatformProcessor_binaryProcessor; jclass c_PlatformTarget; jmethodID m_PlatformTarget_inLongOutLong; @@ -334,10 +308,8 @@ namespace ignite static void SetConsoleHandler(ConsoleWriteHandler consoleHandler); static int RemoveConsoleHandler(ConsoleWriteHandler consoleHandler); - jobject IgnitionStart(char* cfgPath, char* name, int factoryId, long long dataPtr); - jobject IgnitionStart(char* cfgPath, char* name, int factoryId, long long dataPtr, JniErrorInfo* errInfo); - jobject IgnitionInstance(char* name); - jobject IgnitionInstance(char* name, JniErrorInfo* errInfo); + void IgnitionStart(char* cfgPath, char* name, int factoryId, long long dataPtr); + void IgnitionStart(char* cfgPath, char* name, int factoryId, long long dataPtr, JniErrorInfo* errInfo); long long IgnitionEnvironmentPointer(char* name); long long IgnitionEnvironmentPointer(char* name, JniErrorInfo* errInfo); bool IgnitionStop(char* name, bool cancel); @@ -345,41 +317,6 @@ namespace ignite void IgnitionStopAll(bool cancel); void IgnitionStopAll(bool cancel, JniErrorInfo* errInfo); - void ProcessorReleaseStart(jobject obj); - jobject ProcessorProjection(jobject obj, JniErrorInfo* errInfo = NULL); - jobject ProcessorCache(jobject obj, const char* name); - jobject ProcessorCache(jobject obj, const char* name, JniErrorInfo* errInfo); - jobject ProcessorCreateCache(jobject obj, const char* name); - jobject ProcessorCreateCache(jobject obj, const char* name, JniErrorInfo* errInfo); - jobject ProcessorGetOrCreateCache(jobject obj, const char* name); - jobject ProcessorGetOrCreateCache(jobject obj, const char* name, JniErrorInfo* errInfo); - jobject ProcessorCreateCacheFromConfig(jobject obj, long long memPtr); - jobject ProcessorCreateCacheFromConfig(jobject obj, long long memPtr, JniErrorInfo* errInfo); - jobject ProcessorGetOrCreateCacheFromConfig(jobject obj, long long memPtr); - jobject ProcessorGetOrCreateCacheFromConfig(jobject obj, long long memPtr, JniErrorInfo* errInfo); - jobject ProcessorCreateNearCache(jobject obj, const char* name, long long memPtr); - jobject ProcessorGetOrCreateNearCache(jobject obj, const char* name, long long memPtr); - void ProcessorDestroyCache(jobject obj, const char* name); - void ProcessorDestroyCache(jobject obj, const char* name, JniErrorInfo* errInfo); - jobject ProcessorAffinity(jobject obj, const char* name); - jobject ProcessorDataStreamer(jobject obj, const char* name, bool keepPortable); - jobject ProcessorTransactions(jobject obj, JniErrorInfo* errInfo = NULL); - jobject ProcessorCompute(jobject obj, jobject prj); - jobject ProcessorCompute(jobject obj, jobject prj, JniErrorInfo* errInfo); - jobject ProcessorMessage(jobject obj, jobject prj); - jobject ProcessorEvents(jobject obj, jobject prj); - jobject ProcessorServices(jobject obj, jobject prj); - jobject ProcessorExtensions(jobject obj); - jobject ProcessorExtension(jobject obj, int id); - jobject ProcessorAtomicLong(jobject obj, char* name, long long initVal, bool create); - jobject ProcessorAtomicSequence(jobject obj, char* name, long long initVal, bool create); - jobject ProcessorAtomicReference(jobject obj, char* name, long long memPtr, bool create); - void ProcessorGetIgniteConfiguration(jobject obj, long long memPtr); - void ProcessorGetCacheNames(jobject obj, long long memPtr); - bool ProcessorLoggerIsLevelEnabled(jobject obj, int level); - void ProcessorLoggerLog(jobject obj, int level, char* message, char* category, char* errorInfo); - jobject ProcessorBinaryProcessor(jobject obj); - long long TargetInLongOutLong(jobject obj, int type, long long memPtr, JniErrorInfo* errInfo = NULL); long long TargetInStreamOutLong(jobject obj, int type, long long memPtr, JniErrorInfo* errInfo = NULL); void TargetInStreamOutStream(jobject obj, int opType, long long inMemPtr, long long outMemPtr, JniErrorInfo* errInfo = NULL); @@ -406,9 +343,6 @@ namespace ignite void ExceptionCheck(JNIEnv* env); void ExceptionCheck(JNIEnv* env, JniErrorInfo* errInfo); jobject LocalToGlobal(JNIEnv* env, jobject obj); - jobject ProcessorCache0(jobject proc, const char* name, jmethodID mthd, JniErrorInfo* errInfo); - jobject ProcessorCacheFromConfig0(jobject proc, long long memPtr, jmethodID mthd, JniErrorInfo* errInfo); - jobject ProcessorGetOrCreateNearCache0(jobject obj, const char* name, long long memPtr, jmethodID methodID); }; JNIEXPORT jlong JNICALL JniCacheStoreCreate(JNIEnv *env, jclass cls, jlong envPtr, jlong memPtr); diff --git a/modules/platforms/cpp/jni/project/vs/module.def b/modules/platforms/cpp/jni/project/vs/module.def index 82cc41e97b882..53e7e423f7a17 100644 --- a/modules/platforms/cpp/jni/project/vs/module.def +++ b/modules/platforms/cpp/jni/project/vs/module.def @@ -2,19 +2,9 @@ LIBRARY ignite.jni.dll EXPORTS IgniteReallocate @1 IgniteIgnitionStart @2 -IgniteIgnitionInstance @3 IgniteIgnitionEnvironmentPointer @4 IgniteIgnitionStop @5 IgniteIgnitionStopAll @6 -IgniteProcessorReleaseStart @8 -IgniteProcessorProjection @9 -IgniteProcessorCache @10 -IgniteProcessorCreateCache @11 -IgniteProcessorGetOrCreateCache @12 -IgniteProcessorAffinity @13 -IgniteProcessorDataStreamer @14 -IgniteProcessorTransactions @15 -IgniteProcessorServices @16 IgniteTargetInStreamOutObject @17 IgniteTargetInStreamOutLong @18 IgniteTargetOutStream @19 @@ -22,9 +12,6 @@ IgniteTargetInStreamOutStream @20 IgniteTargetInObjectStreamOutObjectStream @21 IgniteTargetInLongOutLong @24 IgniteTargetInStreamAsync @25 -IgniteProcessorCompute @64 -IgniteProcessorMessage @65 -IgniteProcessorEvents @66 IgniteAcquire @80 IgniteRelease @81 IgniteThrowToJava @82 @@ -33,20 +20,5 @@ IgniteCreateContext @84 IgniteDeleteContext @85 IgniteDestroyJvm @86 IgniteTargetOutObject @91 -IgniteProcessorExtension @96 -IgniteProcessorExtensions @97 -IgniteProcessorAtomicLong @98 -IgniteProcessorCreateCacheFromConfig @114 -IgniteProcessorGetOrCreateCacheFromConfig @115 -IgniteProcessorGetIgniteConfiguration @116 -IgniteProcessorDestroyCache @117 -IgniteProcessorAtomicSequence @118 -IgniteProcessorAtomicReference @128 -IgniteProcessorCreateNearCache @131 -IgniteProcessorGetOrCreateNearCache @132 -IgniteProcessorGetCacheNames @133 IgniteSetConsoleHandler @135 -IgniteRemoveConsoleHandler @136 -IgniteProcessorLoggerIsLevelEnabled @137 -IgniteProcessorLoggerLog @138 -IgniteProcessorBinaryProcessor @139 \ No newline at end of file +IgniteRemoveConsoleHandler @136 \ No newline at end of file diff --git a/modules/platforms/cpp/jni/src/exports.cpp b/modules/platforms/cpp/jni/src/exports.cpp index 2aecd59af73f1..9b7defd52dd18 100644 --- a/modules/platforms/cpp/jni/src/exports.cpp +++ b/modules/platforms/cpp/jni/src/exports.cpp @@ -26,12 +26,8 @@ extern "C" { return gcj::JniContext::Reallocate(memPtr, cap); } - void* IGNITE_CALL IgniteIgnitionStart(gcj::JniContext* ctx, char* cfgPath, char* name, int factoryId, long long dataPtr) { - return ctx->IgnitionStart(cfgPath, name, factoryId, dataPtr); - } - - void* IGNITE_CALL IgniteIgnitionInstance(gcj::JniContext* ctx, char* name) { - return ctx->IgnitionInstance(name); + void IGNITE_CALL IgniteIgnitionStart(gcj::JniContext* ctx, char* cfgPath, char* name, int factoryId, long long dataPtr) { + ctx->IgnitionStart(cfgPath, name, factoryId, dataPtr); } long long IGNITE_CALL IgniteIgnitionEnvironmentPointer(gcj::JniContext* ctx, char* name) { @@ -46,118 +42,10 @@ extern "C" { return ctx->IgnitionStopAll(cancel); } - void IGNITE_CALL IgniteProcessorReleaseStart(gcj::JniContext* ctx, void* obj) { - return ctx->ProcessorReleaseStart(static_cast(obj)); - } - - void* IGNITE_CALL IgniteProcessorProjection(gcj::JniContext* ctx, void* obj) { - return ctx->ProcessorProjection(static_cast(obj)); - } - - void* IGNITE_CALL IgniteProcessorCache(gcj::JniContext* ctx, void* obj, char* name) { - return ctx->ProcessorCache(static_cast(obj), name); - } - - void* IGNITE_CALL IgniteProcessorCreateCache(gcj::JniContext* ctx, void* obj, char* name) { - return ctx->ProcessorCreateCache(static_cast(obj), name); - } - - void* IGNITE_CALL IgniteProcessorGetOrCreateCache(gcj::JniContext* ctx, void* obj, char* name) { - return ctx->ProcessorGetOrCreateCache(static_cast(obj), name); - } - - void* IGNITE_CALL IgniteProcessorCreateCacheFromConfig(gcj::JniContext* ctx, void* obj, long long memPtr) { - return ctx->ProcessorCreateCacheFromConfig(static_cast(obj), memPtr); - } - - void* IGNITE_CALL IgniteProcessorGetOrCreateCacheFromConfig(gcj::JniContext* ctx, void* obj, long long memPtr) { - return ctx->ProcessorGetOrCreateCacheFromConfig(static_cast(obj), memPtr); - } - - void* IGNITE_CALL IgniteProcessorCreateNearCache(gcj::JniContext* ctx, void* obj, char* name, long long memPtr) { - return ctx->ProcessorCreateNearCache(static_cast(obj), name, memPtr); - } - - void* IGNITE_CALL IgniteProcessorGetOrCreateNearCache(gcj::JniContext* ctx, void* obj, char* name, long long memPtr) { - return ctx->ProcessorGetOrCreateNearCache(static_cast(obj), name, memPtr); - } - - void IGNITE_CALL IgniteProcessorDestroyCache(gcj::JniContext* ctx, void* obj, char* name) { - ctx->ProcessorDestroyCache(static_cast(obj), name); - } - - void* IGNITE_CALL IgniteProcessorAffinity(gcj::JniContext* ctx, void* obj, char* name) { - return ctx->ProcessorAffinity(static_cast(obj), name); - } - - void*IGNITE_CALL IgniteProcessorDataStreamer(gcj::JniContext* ctx, void* obj, char* name, bool keepPortable) { - return ctx->ProcessorDataStreamer(static_cast(obj), name, keepPortable); - } - - void* IGNITE_CALL IgniteProcessorTransactions(gcj::JniContext* ctx, void* obj) { - return ctx->ProcessorTransactions(static_cast(obj)); - } - - void* IGNITE_CALL IgniteProcessorCompute(gcj::JniContext* ctx, void* obj, void* prj) { - return ctx->ProcessorCompute(static_cast(obj), static_cast(prj)); - } - - void* IGNITE_CALL IgniteProcessorMessage(gcj::JniContext* ctx, void* obj, void* prj) { - return ctx->ProcessorMessage(static_cast(obj), static_cast(prj)); - } - - void* IGNITE_CALL IgniteProcessorEvents(gcj::JniContext* ctx, void* obj, void* prj) { - return ctx->ProcessorEvents(static_cast(obj), static_cast(prj)); - } - - void* IGNITE_CALL IgniteProcessorServices(gcj::JniContext* ctx, void* obj, void* prj) { - return ctx->ProcessorServices(static_cast(obj), static_cast(prj)); - } - - void* IGNITE_CALL IgniteProcessorExtensions(gcj::JniContext* ctx, void* obj) { - return ctx->ProcessorExtensions(static_cast(obj)); - } - - void* IGNITE_CALL IgniteProcessorExtension(gcj::JniContext* ctx, void* obj, int id) { - return ctx->ProcessorExtension(static_cast(obj), id); - } - - void* IGNITE_CALL IgniteProcessorAtomicLong(gcj::JniContext* ctx, void* obj, char* name, long long initVal, bool create) { - return ctx->ProcessorAtomicLong(static_cast(obj), name, initVal, create); - } - - void* IGNITE_CALL IgniteProcessorAtomicSequence(gcj::JniContext* ctx, void* obj, char* name, long long initVal, bool create) { - return ctx->ProcessorAtomicSequence(static_cast(obj), name, initVal, create); - } - - void* IGNITE_CALL IgniteProcessorAtomicReference(gcj::JniContext* ctx, void* obj, char* name, long long memPtr, bool create) { - return ctx->ProcessorAtomicReference(static_cast(obj), name, memPtr, create); - } - - void IGNITE_CALL IgniteProcessorGetIgniteConfiguration(gcj::JniContext* ctx, void* obj, long long memPtr) { - return ctx->ProcessorGetIgniteConfiguration(static_cast(obj), memPtr); - } - - void IGNITE_CALL IgniteProcessorGetCacheNames(gcj::JniContext* ctx, void* obj, long long memPtr) { - return ctx->ProcessorGetCacheNames(static_cast(obj), memPtr); - } - long long IGNITE_CALL IgniteTargetInLongOutLong(gcj::JniContext* ctx, void* obj, int opType, long long val) { return ctx->TargetInLongOutLong(static_cast(obj), opType, val); } - bool IGNITE_CALL IgniteProcessorLoggerIsLevelEnabled(gcj::JniContext* ctx, void* obj, int level) { - return ctx->ProcessorLoggerIsLevelEnabled(static_cast(obj), level); - } - - void IGNITE_CALL IgniteProcessorLoggerLog(gcj::JniContext* ctx, void* obj, int level, char* message, char* category, char* errorInfo) { - ctx->ProcessorLoggerLog(static_cast(obj), level, message, category, errorInfo); - } - - void* IGNITE_CALL IgniteProcessorBinaryProcessor(gcj::JniContext* ctx, void* obj) { - return ctx->ProcessorBinaryProcessor(static_cast(obj)); - } - long long IGNITE_CALL IgniteTargetInStreamOutLong(gcj::JniContext* ctx, void* obj, int opType, long long memPtr) { return ctx->TargetInStreamOutLong(static_cast(obj), opType, memPtr); } diff --git a/modules/platforms/cpp/jni/src/java.cpp b/modules/platforms/cpp/jni/src/java.cpp index bc6af34bf286d..7eadec0a27d69 100644 --- a/modules/platforms/cpp/jni/src/java.cpp +++ b/modules/platforms/cpp/jni/src/java.cpp @@ -223,32 +223,6 @@ namespace ignite const char* C_PLATFORM_PROCESSOR = "org/apache/ignite/internal/processors/platform/PlatformProcessor"; JniMethod M_PLATFORM_PROCESSOR_RELEASE_START = JniMethod("releaseStart", "()V", false); - JniMethod M_PLATFORM_PROCESSOR_PROJECTION = JniMethod("projection", "()Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;", false); - JniMethod M_PLATFORM_PROCESSOR_CACHE = JniMethod("cache", "(Ljava/lang/String;)Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;", false); - JniMethod M_PLATFORM_PROCESSOR_CREATE_CACHE = JniMethod("createCache", "(Ljava/lang/String;)Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;", false); - JniMethod M_PLATFORM_PROCESSOR_GET_OR_CREATE_CACHE = JniMethod("getOrCreateCache", "(Ljava/lang/String;)Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;", false); - JniMethod M_PLATFORM_PROCESSOR_CREATE_CACHE_FROM_CONFIG = JniMethod("createCacheFromConfig", "(J)Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;", false); - JniMethod M_PLATFORM_PROCESSOR_GET_OR_CREATE_CACHE_FROM_CONFIG = JniMethod("getOrCreateCacheFromConfig", "(J)Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;", false); - JniMethod M_PLATFORM_PROCESSOR_CREATE_NEAR_CACHE = JniMethod("createNearCache", "(Ljava/lang/String;J)Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;", false); - JniMethod M_PLATFORM_PROCESSOR_GET_OR_CREATE_NEAR_CACHE = JniMethod("getOrCreateNearCache", "(Ljava/lang/String;J)Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;", false); - JniMethod M_PLATFORM_PROCESSOR_DESTROY_CACHE = JniMethod("destroyCache", "(Ljava/lang/String;)V", false); - JniMethod M_PLATFORM_PROCESSOR_AFFINITY = JniMethod("affinity", "(Ljava/lang/String;)Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;", false); - JniMethod M_PLATFORM_PROCESSOR_DATA_STREAMER = JniMethod("dataStreamer", "(Ljava/lang/String;Z)Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;", false); - JniMethod M_PLATFORM_PROCESSOR_TRANSACTIONS = JniMethod("transactions", "()Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;", false); - JniMethod M_PLATFORM_PROCESSOR_COMPUTE = JniMethod("compute", "(Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;)Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;", false); - JniMethod M_PLATFORM_PROCESSOR_MESSAGE = JniMethod("message", "(Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;)Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;", false); - JniMethod M_PLATFORM_PROCESSOR_EVENTS = JniMethod("events", "(Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;)Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;", false); - JniMethod M_PLATFORM_PROCESSOR_SERVICES = JniMethod("services", "(Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;)Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;", false); - JniMethod M_PLATFORM_PROCESSOR_EXTENSIONS = JniMethod("extensions", "()Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;", false); - JniMethod M_PLATFORM_PROCESSOR_EXTENSION = JniMethod("extension", "(I)Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;", false); - JniMethod M_PLATFORM_PROCESSOR_ATOMIC_LONG = JniMethod("atomicLong", "(Ljava/lang/String;JZ)Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;", false); - JniMethod M_PLATFORM_PROCESSOR_ATOMIC_SEQUENCE = JniMethod("atomicSequence", "(Ljava/lang/String;JZ)Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;", false); - JniMethod M_PLATFORM_PROCESSOR_ATOMIC_REFERENCE = JniMethod("atomicReference", "(Ljava/lang/String;JZ)Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;", false); - JniMethod M_PLATFORM_PROCESSOR_GET_IGNITE_CONFIGURATION = JniMethod("getIgniteConfiguration", "(J)V", false); - JniMethod M_PLATFORM_PROCESSOR_GET_CACHE_NAMES = JniMethod("getCacheNames", "(J)V", false); - JniMethod M_PLATFORM_PROCESSOR_LOGGER_IS_LEVEL_ENABLED = JniMethod("loggerIsLevelEnabled", "(I)Z", false); - JniMethod M_PLATFORM_PROCESSOR_LOGGER_LOG = JniMethod("loggerLog", "(ILjava/lang/String;Ljava/lang/String;Ljava/lang/String;)V", false); - JniMethod M_PLATFORM_PROCESSOR_BINARY_PROCESSOR = JniMethod("binaryProcessor", "()Lorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;", false); const char* C_PLATFORM_TARGET = "org/apache/ignite/internal/processors/platform/PlatformTargetProxy"; JniMethod M_PLATFORM_TARGET_IN_LONG_OUT_LONG = JniMethod("inLongOutLong", "(IJ)J", false); @@ -262,86 +236,9 @@ namespace ignite const char* C_PLATFORM_CALLBACK_UTILS = "org/apache/ignite/internal/processors/platform/callback/PlatformCallbackUtils"; - JniMethod M_PLATFORM_CALLBACK_UTILS_CACHE_STORE_CREATE = JniMethod("cacheStoreCreate", "(JJ)J", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_CACHE_STORE_INVOKE = JniMethod("cacheStoreInvoke", "(JJJ)I", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_CACHE_STORE_DESTROY = JniMethod("cacheStoreDestroy", "(JJ)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_CACHE_STORE_SESSION_CREATE = JniMethod("cacheStoreSessionCreate", "(JJ)J", true); - - JniMethod M_PLATFORM_CALLBACK_UTILS_CACHE_ENTRY_FILTER_CREATE = JniMethod("cacheEntryFilterCreate", "(JJ)J", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_CACHE_ENTRY_FILTER_APPLY = JniMethod("cacheEntryFilterApply", "(JJJ)I", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_CACHE_ENTRY_FILTER_DESTROY = JniMethod("cacheEntryFilterDestroy", "(JJ)V", true); - - JniMethod M_PLATFORM_CALLBACK_UTILS_CACHE_INVOKE = JniMethod("cacheInvoke", "(JJJ)V", true); - - JniMethod M_PLATFORM_CALLBACK_UTILS_COMPUTE_TASK_MAP = JniMethod("computeTaskMap", "(JJJJ)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_COMPUTE_TASK_JOB_RESULT = JniMethod("computeTaskJobResult", "(JJJJ)I", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_COMPUTE_TASK_REDUCE = JniMethod("computeTaskReduce", "(JJ)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_COMPUTE_TASK_COMPLETE = JniMethod("computeTaskComplete", "(JJJ)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_COMPUTE_JOB_SERIALIZE = JniMethod("computeJobSerialize", "(JJJ)I", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_COMPUTE_JOB_CREATE = JniMethod("computeJobCreate", "(JJ)J", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_COMPUTE_JOB_EXECUTE = JniMethod("computeJobExecute", "(JJIJ)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_COMPUTE_JOB_DESTROY = JniMethod("computeJobDestroy", "(JJ)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_COMPUTE_JOB_CANCEL = JniMethod("computeJobCancel", "(JJ)V", true); - - JniMethod M_PLATFORM_CALLBACK_UTILS_CONTINUOUS_QUERY_LSNR_APPLY = JniMethod("continuousQueryListenerApply", "(JJJ)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_CONTINUOUS_QUERY_FILTER_CREATE = JniMethod("continuousQueryFilterCreate", "(JJ)J", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_CONTINUOUS_QUERY_FILTER_EVAL = JniMethod("continuousQueryFilterApply", "(JJJ)I", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_CONTINUOUS_QUERY_FILTER_RELEASE = JniMethod("continuousQueryFilterRelease", "(JJ)V", true); - - JniMethod M_PLATFORM_CALLBACK_UTILS_DATA_STREAMER_TOPOLOGY_UPDATE = JniMethod("dataStreamerTopologyUpdate", "(JJJI)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_DATA_STREAMER_STREAM_RECEIVER_INVOKE = JniMethod("dataStreamerStreamReceiverInvoke", "(JJLorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;JZ)V", true); - - JniMethod M_PLATFORM_CALLBACK_UTILS_FUTURE_BYTE_RES = JniMethod("futureByteResult", "(JJI)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_FUTURE_BOOL_RES = JniMethod("futureBoolResult", "(JJI)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_FUTURE_SHORT_RES = JniMethod("futureShortResult", "(JJI)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_FUTURE_CHAR_RES = JniMethod("futureCharResult", "(JJI)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_FUTURE_INT_RES = JniMethod("futureIntResult", "(JJI)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_FUTURE_FLOAT_RES = JniMethod("futureFloatResult", "(JJF)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_FUTURE_LONG_RES = JniMethod("futureLongResult", "(JJJ)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_FUTURE_DOUBLE_RES = JniMethod("futureDoubleResult", "(JJD)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_FUTURE_OBJ_RES = JniMethod("futureObjectResult", "(JJJ)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_FUTURE_NULL_RES = JniMethod("futureNullResult", "(JJ)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_FUTURE_ERR = JniMethod("futureError", "(JJJ)V", true); - - JniMethod M_PLATFORM_CALLBACK_UTILS_LIFECYCLE_EVENT = JniMethod("lifecycleEvent", "(JJI)V", true); - - JniMethod M_PLATFORM_CALLBACK_UTILS_MESSAGING_FILTER_CREATE = JniMethod("messagingFilterCreate", "(JJ)J", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_MESSAGING_FILTER_APPLY = JniMethod("messagingFilterApply", "(JJJ)I", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_MESSAGING_FILTER_DESTROY = JniMethod("messagingFilterDestroy", "(JJ)V", true); - - JniMethod M_PLATFORM_CALLBACK_UTILS_EVENT_FILTER_CREATE = JniMethod("eventFilterCreate", "(JJ)J", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_EVENT_FILTER_APPLY = JniMethod("eventFilterApply", "(JJJ)I", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_EVENT_FILTER_DESTROY = JniMethod("eventFilterDestroy", "(JJ)V", true); - - JniMethod M_PLATFORM_CALLBACK_UTILS_SERVICE_INIT = JniMethod("serviceInit", "(JJ)J", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_SERVICE_EXECUTE = JniMethod("serviceExecute", "(JJJ)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_SERVICE_CANCEL = JniMethod("serviceCancel", "(JJJ)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_SERVICE_INVOKE_METHOD = JniMethod("serviceInvokeMethod", "(JJJJ)V", true); - - JniMethod M_PLATFORM_CALLBACK_UTILS_CLUSTER_NODE_FILTER_APPLY = JniMethod("clusterNodeFilterApply", "(JJ)I", true); - - JniMethod M_PLATFORM_CALLBACK_UTILS_NODE_INFO = JniMethod("nodeInfo", "(JJ)V", true); - - JniMethod M_PLATFORM_CALLBACK_UTILS_MEMORY_REALLOCATE = JniMethod("memoryReallocate", "(JJI)V", true); - - JniMethod M_PLATFORM_CALLBACK_UTILS_ON_START = JniMethod("onStart", "(JLjava/lang/Object;J)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_ON_STOP = JniMethod("onStop", "(J)V", true); - - JniMethod M_PLATFORM_CALLBACK_UTILS_EXTENSION_CALLBACK_IN_LONG_OUT_LONG = JniMethod("extensionCallbackInLongOutLong", "(JIJ)J", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_EXTENSION_CALLBACK_IN_LONG_LONG_OUT_LONG = JniMethod("extensionCallbackInLongLongOutLong", "(JIJJ)J", true); - - JniMethod M_PLATFORM_CALLBACK_UTILS_ON_CLIENT_DISCONNECTED = JniMethod("onClientDisconnected", "(J)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_ON_CLIENT_RECONNECTED = JniMethod("onClientReconnected", "(JZ)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_LOGGER_LOG = JniMethod("loggerLog", "(JILjava/lang/String;Ljava/lang/String;Ljava/lang/String;J)V", true); JniMethod M_PLATFORM_CALLBACK_UTILS_LOGGER_IS_LEVEL_ENABLED = JniMethod("loggerIsLevelEnabled", "(JI)Z", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_AFFINITY_FUNCTION_INIT = JniMethod("affinityFunctionInit", "(JJLorg/apache/ignite/internal/processors/platform/PlatformTargetProxy;)J", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_AFFINITY_FUNCTION_PARTITION = JniMethod("affinityFunctionPartition", "(JJJ)I", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_AFFINITY_FUNCTION_ASSIGN_PARTITIONS = JniMethod("affinityFunctionAssignPartitions", "(JJJJ)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_AFFINITY_FUNCTION_REMOVE_NODE = JniMethod("affinityFunctionRemoveNode", "(JJJ)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_AFFINITY_FUNCTION_DESTROY = JniMethod("affinityFunctionDestroy", "(JJ)V", true); - JniMethod M_PLATFORM_CALLBACK_UTILS_CONSOLE_WRITE = JniMethod("consoleWrite", "(Ljava/lang/String;Z)V", true); JniMethod M_PLATFORM_CALLBACK_UTILS_IN_LONG_OUT_LONG = JniMethod("inLongOutLong", "(JIJ)J", true); @@ -353,7 +250,7 @@ namespace ignite JniMethod M_PLATFORM_UTILS_GET_FULL_STACK_TRACE = JniMethod("getFullStackTrace", "(Ljava/lang/Throwable;)Ljava/lang/String;", true); const char* C_PLATFORM_IGNITION = "org/apache/ignite/internal/processors/platform/PlatformIgnition"; - JniMethod M_PLATFORM_IGNITION_START = JniMethod("start", "(Ljava/lang/String;Ljava/lang/String;IJJ)Lorg/apache/ignite/internal/processors/platform/PlatformProcessor;", true); + JniMethod M_PLATFORM_IGNITION_START = JniMethod("start", "(Ljava/lang/String;Ljava/lang/String;IJJ)V", true); JniMethod M_PLATFORM_IGNITION_INSTANCE = JniMethod("instance", "(Ljava/lang/String;)Lorg/apache/ignite/internal/processors/platform/PlatformProcessor;", true); JniMethod M_PLATFORM_IGNITION_ENVIRONMENT_POINTER = JniMethod("environmentPointer", "(Ljava/lang/String;)J", true); JniMethod M_PLATFORM_IGNITION_STOP = JniMethod("stop", "(Ljava/lang/String;Z)Z", true); @@ -554,32 +451,6 @@ namespace ignite c_PlatformProcessor = FindClass(env, C_PLATFORM_PROCESSOR); m_PlatformProcessor_releaseStart = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_RELEASE_START); - m_PlatformProcessor_cache = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_CACHE); - m_PlatformProcessor_createCache = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_CREATE_CACHE); - m_PlatformProcessor_getOrCreateCache = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_GET_OR_CREATE_CACHE); - m_PlatformProcessor_createCacheFromConfig = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_CREATE_CACHE_FROM_CONFIG); - m_PlatformProcessor_getOrCreateCacheFromConfig = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_GET_OR_CREATE_CACHE_FROM_CONFIG); - m_PlatformProcessor_createNearCache = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_CREATE_NEAR_CACHE); - m_PlatformProcessor_getOrCreateNearCache = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_GET_OR_CREATE_NEAR_CACHE); - m_PlatformProcessor_destroyCache = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_DESTROY_CACHE); - m_PlatformProcessor_affinity = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_AFFINITY); - m_PlatformProcessor_dataStreamer = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_DATA_STREAMER); - m_PlatformProcessor_transactions = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_TRANSACTIONS); - m_PlatformProcessor_projection = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_PROJECTION); - m_PlatformProcessor_compute = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_COMPUTE); - m_PlatformProcessor_message = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_MESSAGE); - m_PlatformProcessor_events = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_EVENTS); - m_PlatformProcessor_services = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_SERVICES); - m_PlatformProcessor_extensions = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_EXTENSIONS); - m_PlatformProcessor_extension = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_EXTENSION); - m_PlatformProcessor_atomicLong = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_ATOMIC_LONG); - m_PlatformProcessor_atomicSequence = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_ATOMIC_SEQUENCE); - m_PlatformProcessor_atomicReference = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_ATOMIC_REFERENCE); - m_PlatformProcessor_getIgniteConfiguration = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_GET_IGNITE_CONFIGURATION); - m_PlatformProcessor_getCacheNames = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_GET_CACHE_NAMES); - m_PlatformProcessor_loggerIsLevelEnabled = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_LOGGER_IS_LEVEL_ENABLED); - m_PlatformProcessor_loggerLog = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_LOGGER_LOG); - m_PlatformProcessor_binaryProcessor = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_BINARY_PROCESSOR); c_PlatformTarget = FindClass(env, C_PLATFORM_TARGET); m_PlatformTarget_inLongOutLong = FindMethod(env, c_PlatformTarget, M_PLATFORM_TARGET_IN_LONG_OUT_LONG); @@ -871,18 +742,18 @@ namespace ignite } } - jobject JniContext::IgnitionStart(char* cfgPath, char* name, int factoryId, long long dataPtr) { + void JniContext::IgnitionStart(char* cfgPath, char* name, int factoryId, long long dataPtr) { return IgnitionStart(cfgPath, name, factoryId, dataPtr, NULL); } - jobject JniContext::IgnitionStart(char* cfgPath, char* name, int factoryId, long long dataPtr, JniErrorInfo* errInfo) + void JniContext::IgnitionStart(char* cfgPath, char* name, int factoryId, long long dataPtr, JniErrorInfo* errInfo) { JNIEnv* env = Attach(); jstring cfgPath0 = env->NewStringUTF(cfgPath); jstring name0 = env->NewStringUTF(name); - jobject interop = env->CallStaticObjectMethod( + env->CallStaticVoidMethod( jvm->GetMembers().c_PlatformIgnition, jvm->GetMembers().m_PlatformIgnition_start, cfgPath0, @@ -893,30 +764,8 @@ namespace ignite ); ExceptionCheck(env, errInfo); - - return LocalToGlobal(env, interop); - } - - - jobject JniContext::IgnitionInstance(char* name) - { - return IgnitionInstance(name, NULL); - } - - jobject JniContext::IgnitionInstance(char* name, JniErrorInfo* errInfo) - { - JNIEnv* env = Attach(); - - jstring name0 = env->NewStringUTF(name); - - jobject interop = env->CallStaticObjectMethod(jvm->GetMembers().c_PlatformIgnition, - jvm->GetMembers().m_PlatformIgnition_instance, name0); - - ExceptionCheck(env, errInfo); - - return LocalToGlobal(env, interop); } - + long long JniContext::IgnitionEnvironmentPointer(char* name) { return IgnitionEnvironmentPointer(name, NULL); @@ -970,318 +819,6 @@ namespace ignite ExceptionCheck(env, errInfo); } - void JniContext::ProcessorReleaseStart(jobject obj) { - JNIEnv* env = Attach(); - - env->CallVoidMethod(obj, jvm->GetMembers().m_PlatformProcessor_releaseStart); - - ExceptionCheck(env); - } - - jobject JniContext::ProcessorProjection(jobject obj, JniErrorInfo* errInfo) { - JNIEnv* env = Attach(); - - jobject prj = env->CallObjectMethod(obj, jvm->GetMembers().m_PlatformProcessor_projection); - - ExceptionCheck(env, errInfo); - - return LocalToGlobal(env, prj); - } - - jobject JniContext::ProcessorCache0(jobject obj, const char* name, jmethodID mthd, JniErrorInfo* errInfo) - { - JNIEnv* env = Attach(); - - jstring name0 = name != NULL ? env->NewStringUTF(name) : NULL; - - jobject cache = env->CallObjectMethod(obj, mthd, name0); - - if (name0) - env->DeleteLocalRef(name0); - - ExceptionCheck(env, errInfo); - - return LocalToGlobal(env, cache); - } - - jobject JniContext::ProcessorCacheFromConfig0(jobject obj, long long memPtr, jmethodID mthd, JniErrorInfo* errInfo) - { - JNIEnv* env = Attach(); - - jobject cache = env->CallObjectMethod(obj, mthd, memPtr); - - ExceptionCheck(env, errInfo); - - return LocalToGlobal(env, cache); - } - - jobject JniContext::ProcessorCache(jobject obj, const char* name) { - return ProcessorCache(obj, name, NULL); - } - - jobject JniContext::ProcessorCache(jobject obj, const char* name, JniErrorInfo* errInfo) { - return ProcessorCache0(obj, name, jvm->GetMembers().m_PlatformProcessor_cache, errInfo); - } - - jobject JniContext::ProcessorCreateCache(jobject obj, const char* name) { - return ProcessorCreateCache(obj, name, NULL); - } - - jobject JniContext::ProcessorCreateCache(jobject obj, const char* name, JniErrorInfo* errInfo) - { - return ProcessorCache0(obj, name, jvm->GetMembers().m_PlatformProcessor_createCache, errInfo); - } - - jobject JniContext::ProcessorGetOrCreateCache(jobject obj, const char* name) { - return ProcessorGetOrCreateCache(obj, name, NULL); - } - - jobject JniContext::ProcessorGetOrCreateCache(jobject obj, const char* name, JniErrorInfo* errInfo) - { - return ProcessorCache0(obj, name, jvm->GetMembers().m_PlatformProcessor_getOrCreateCache, errInfo); - } - - void JniContext::ProcessorDestroyCache(jobject obj, const char* name) { - ProcessorDestroyCache(obj, name, NULL); - } - - void JniContext::ProcessorDestroyCache(jobject obj, const char* name, JniErrorInfo* errInfo) - { - JNIEnv* env = Attach(); - - jstring name0 = name != NULL ? env->NewStringUTF(name) : NULL; - - env->CallVoidMethod(obj, jvm->GetMembers().m_PlatformProcessor_destroyCache, name0); - - if (name0) - env->DeleteLocalRef(name0); - - ExceptionCheck(env, errInfo); - } - - jobject JniContext::ProcessorCreateCacheFromConfig(jobject obj, long long memPtr) { - return ProcessorCreateCacheFromConfig(obj, memPtr, NULL); - } - - jobject JniContext::ProcessorCreateCacheFromConfig(jobject obj, long long memPtr, JniErrorInfo* errInfo) - { - return ProcessorCacheFromConfig0(obj, memPtr, jvm->GetMembers().m_PlatformProcessor_createCacheFromConfig, errInfo); - } - - jobject JniContext::ProcessorGetOrCreateCacheFromConfig(jobject obj, long long memPtr) { - return ProcessorGetOrCreateCacheFromConfig(obj, memPtr, NULL); - } - - jobject JniContext::ProcessorGetOrCreateCacheFromConfig(jobject obj, long long memPtr, JniErrorInfo* errInfo) - { - return ProcessorCacheFromConfig0(obj, memPtr, jvm->GetMembers().m_PlatformProcessor_getOrCreateCacheFromConfig, errInfo); - } - - jobject JniContext::ProcessorCreateNearCache(jobject obj, const char* name, long long memPtr) - { - return ProcessorGetOrCreateNearCache0(obj, name, memPtr, jvm->GetMembers().m_PlatformProcessor_createNearCache); - } - - jobject JniContext::ProcessorGetOrCreateNearCache(jobject obj, const char* name, long long memPtr) - { - return ProcessorGetOrCreateNearCache0(obj, name, memPtr, jvm->GetMembers().m_PlatformProcessor_getOrCreateNearCache); - } - - jobject JniContext::ProcessorGetOrCreateNearCache0(jobject obj, const char* name, long long memPtr, jmethodID methodID) - { - JNIEnv* env = Attach(); - - jstring name0 = name != NULL ? env->NewStringUTF(name) : NULL; - - jobject cache = env->CallObjectMethod(obj, methodID, name0, memPtr); - - if (name0) - env->DeleteLocalRef(name0); - - ExceptionCheck(env); - - return LocalToGlobal(env, cache); - } - - jobject JniContext::ProcessorAffinity(jobject obj, const char* name) { - JNIEnv* env = Attach(); - - jstring name0 = name != NULL ? env->NewStringUTF(name) : NULL; - - jobject aff = env->CallObjectMethod(obj, jvm->GetMembers().m_PlatformProcessor_affinity, name0); - - if (name0) - env->DeleteLocalRef(name0); - - ExceptionCheck(env); - - return LocalToGlobal(env, aff); - } - - jobject JniContext::ProcessorDataStreamer(jobject obj, const char* name, bool keepPortable) { - JNIEnv* env = Attach(); - - jstring name0 = name != NULL ? env->NewStringUTF(name) : NULL; - - jobject ldr = env->CallObjectMethod(obj, jvm->GetMembers().m_PlatformProcessor_dataStreamer, name0, - keepPortable); - - if (name0) - env->DeleteLocalRef(name0); - - ExceptionCheck(env); - - return LocalToGlobal(env, ldr); - } - - jobject JniContext::ProcessorTransactions(jobject obj, JniErrorInfo* errInfo) { - JNIEnv* env = Attach(); - - jobject tx = env->CallObjectMethod(obj, jvm->GetMembers().m_PlatformProcessor_transactions); - - ExceptionCheck(env, errInfo); - - return LocalToGlobal(env, tx); - } - - jobject JniContext::ProcessorCompute(jobject obj, jobject prj) { - JNIEnv* env = Attach(); - - jobject res = env->CallObjectMethod(obj, jvm->GetMembers().m_PlatformProcessor_compute, prj); - - ExceptionCheck(env); - - return LocalToGlobal(env, res); - } - - jobject JniContext::ProcessorCompute(jobject obj, jobject prj, JniErrorInfo* errInfo) { - JNIEnv* env = Attach(); - - jobject res = env->CallObjectMethod(obj, jvm->GetMembers().m_PlatformProcessor_compute, prj); - - ExceptionCheck(env, errInfo); - - return LocalToGlobal(env, res); - } - - jobject JniContext::ProcessorMessage(jobject obj, jobject prj) { - JNIEnv* env = Attach(); - - jobject res = env->CallObjectMethod(obj, jvm->GetMembers().m_PlatformProcessor_message, prj); - - ExceptionCheck(env); - - return LocalToGlobal(env, res); - } - - jobject JniContext::ProcessorEvents(jobject obj, jobject prj) { - JNIEnv* env = Attach(); - - jobject res = env->CallObjectMethod(obj, jvm->GetMembers().m_PlatformProcessor_events, prj); - - ExceptionCheck(env); - - return LocalToGlobal(env, res); - } - - jobject JniContext::ProcessorServices(jobject obj, jobject prj) { - JNIEnv* env = Attach(); - - jobject res = env->CallObjectMethod(obj, jvm->GetMembers().m_PlatformProcessor_services, prj); - - ExceptionCheck(env); - - return LocalToGlobal(env, res); - } - - jobject JniContext::ProcessorExtensions(jobject obj) - { - JNIEnv* env = Attach(); - - jobject res = env->CallObjectMethod(obj, jvm->GetMembers().m_PlatformProcessor_extensions); - - ExceptionCheck(env); - - return LocalToGlobal(env, res); - } - - jobject JniContext::ProcessorExtension(jobject obj, int id) - { - JNIEnv* env = Attach(); - - jobject res = env->CallObjectMethod(obj, jvm->GetMembers().m_PlatformProcessor_extension, id); - - ExceptionCheck(env); - - return LocalToGlobal(env, res); - } - - jobject JniContext::ProcessorAtomicLong(jobject obj, char* name, long long initVal, bool create) - { - JNIEnv* env = Attach(); - - jstring name0 = name != NULL ? env->NewStringUTF(name) : NULL; - - jobject res = env->CallObjectMethod(obj, jvm->GetMembers().m_PlatformProcessor_atomicLong, name0, initVal, create); - - if (name0) - env->DeleteLocalRef(name0); - - ExceptionCheck(env); - - return LocalToGlobal(env, res); - } - - jobject JniContext::ProcessorAtomicSequence(jobject obj, char* name, long long initVal, bool create) - { - JNIEnv* env = Attach(); - - jstring name0 = name != NULL ? env->NewStringUTF(name) : NULL; - - jobject res = env->CallObjectMethod(obj, jvm->GetMembers().m_PlatformProcessor_atomicSequence, name0, initVal, create); - - if (name0) - env->DeleteLocalRef(name0); - - ExceptionCheck(env); - - return LocalToGlobal(env, res); - } - - jobject JniContext::ProcessorAtomicReference(jobject obj, char* name, long long memPtr, bool create) - { - JNIEnv* env = Attach(); - - jstring name0 = name != NULL ? env->NewStringUTF(name) : NULL; - - jobject res = env->CallObjectMethod(obj, jvm->GetMembers().m_PlatformProcessor_atomicReference, name0, memPtr, create); - - if (name0) - env->DeleteLocalRef(name0); - - ExceptionCheck(env); - - return LocalToGlobal(env, res); - } - - void JniContext::ProcessorGetIgniteConfiguration(jobject obj, long long memPtr) - { - JNIEnv* env = Attach(); - - env->CallVoidMethod(obj, jvm->GetMembers().m_PlatformProcessor_getIgniteConfiguration, memPtr); - - ExceptionCheck(env); - } - - void JniContext::ProcessorGetCacheNames(jobject obj, long long memPtr) - { - JNIEnv* env = Attach(); - - env->CallVoidMethod(obj, jvm->GetMembers().m_PlatformProcessor_getCacheNames, memPtr); - - ExceptionCheck(env); - } - long long JniContext::TargetInLongOutLong(jobject obj, int opType, long long val, JniErrorInfo* err) { JNIEnv* env = Attach(); @@ -1292,51 +829,6 @@ namespace ignite return res; } - bool JniContext::ProcessorLoggerIsLevelEnabled(jobject obj, int level) - { - JNIEnv* env = Attach(); - - jboolean res = env->CallBooleanMethod(obj, jvm->GetMembers().m_PlatformProcessor_loggerIsLevelEnabled, level); - - ExceptionCheck(env); - - return res != 0; - } - - void JniContext::ProcessorLoggerLog(jobject obj, int level, char* message, char* category, char* errorInfo) - { - JNIEnv* env = Attach(); - - jstring message0 = message != NULL ? env->NewStringUTF(message) : NULL; - jstring category0 = category != NULL ? env->NewStringUTF(category) : NULL; - jstring errorInfo0 = errorInfo != NULL ? env->NewStringUTF(errorInfo) : NULL; - - - env->CallVoidMethod(obj, jvm->GetMembers().m_PlatformProcessor_loggerLog, level, message0, category0, errorInfo0); - - if (message0) - env->DeleteLocalRef(message0); - - if (category0) - env->DeleteLocalRef(category0); - - if (errorInfo0) - env->DeleteLocalRef(errorInfo0); - - ExceptionCheck(env); - } - - jobject JniContext::ProcessorBinaryProcessor(jobject obj) - { - JNIEnv* env = Attach(); - - jobject res = env->CallObjectMethod(obj, jvm->GetMembers().m_PlatformProcessor_binaryProcessor); - - ExceptionCheck(env); - - return LocalToGlobal(env, res); - } - long long JniContext::TargetInStreamOutLong(jobject obj, int opType, long long memPtr, JniErrorInfo* err) { JNIEnv* env = Attach(); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Ignition.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Ignition.cs index c5d056d6ed5f2..44ebef3b497ad 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Ignition.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Ignition.cs @@ -253,7 +253,9 @@ public static unsafe IIgnite Start(IgniteConfiguration cfg) var javaLogger = log as JavaLogger; if (javaLogger != null) - javaLogger.SetProcessor(interopProc); + { + javaLogger.SetIgnite(node); + } // 6. On-start callback (notify lifecycle components). node.OnStart(); @@ -291,10 +293,14 @@ public static unsafe IIgnite Start(IgniteConfiguration cfg) } finally { + var ignite = _startup.Ignite; + _startup = null; - if (interopProc != null) - UU.ProcessorReleaseStart(interopProc); + if (ignite != null) + { + ignite.ProcessorReleaseStart(); + } } } } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs index b1bf5eb71edad..e6b2408333456 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs @@ -1166,7 +1166,7 @@ IEnumerator IEnumerable.GetEnumerator() { if (loc) { - var target = DoOutOpObject((int) CacheOp.LocIterator, w => w.WriteInt(peekModes)); + var target = DoOutOpObject((int) CacheOp.LocIterator, (IBinaryStream s) => s.WriteInt(peekModes)); return new CacheEnumerator(target, Marshaller, _flagKeepBinary); } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cluster/ClusterGroupImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cluster/ClusterGroupImpl.cs index 6e07b7819b67b..30afe5785b9ff 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cluster/ClusterGroupImpl.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cluster/ClusterGroupImpl.cs @@ -135,6 +135,18 @@ internal class ClusterGroupImpl : PlatformTarget, IClusterGroup /** */ private const int OpGetPersistentStoreMetrics = 30; + /** */ + private const int OpGetCompute = 31; + + /** */ + private const int OpGetMessaging = 32; + + /** */ + private const int OpGetEvents = 33; + + /** */ + private const int OpGetServices = 34; + /** Initial Ignite instance. */ private readonly Ignite _ignite; @@ -147,17 +159,14 @@ internal class ClusterGroupImpl : PlatformTarget, IClusterGroup /** Nodes for the given topology version. */ private volatile IList _nodes; - /** Processor. */ - private readonly IUnmanagedTarget _proc; - /** Compute. */ - private readonly Lazy _comp; + private readonly Lazy _comp; /** Messaging. */ - private readonly Lazy _msg; + private readonly Lazy _msg; /** Events. */ - private readonly Lazy _events; + private readonly Lazy _events; /** Services. */ private readonly Lazy _services; @@ -165,29 +174,20 @@ internal class ClusterGroupImpl : PlatformTarget, IClusterGroup /// /// Constructor. /// - /// Processor. /// Target. - /// Marshaller. /// Grid. /// Predicate. [SuppressMessage("Microsoft.Performance", "CA1805:DoNotInitializeUnnecessarily")] - public ClusterGroupImpl(IUnmanagedTarget proc, IUnmanagedTarget target, Marshaller marsh, - Ignite ignite, Func pred) - : base(target, marsh) + public ClusterGroupImpl(IUnmanagedTarget target, Ignite ignite, Func pred) + : base(target, ignite.Marshaller) { - _proc = proc; _ignite = ignite; _pred = pred; - _comp = new Lazy(() => - new Compute(new ComputeImpl(UU.ProcessorCompute(proc, target), marsh, this, false))); - - _msg = new Lazy(() => new Messaging(UU.ProcessorMessage(proc, target), marsh, this)); - - _events = new Lazy(() => new Events(UU.ProcessorEvents(proc, target), marsh, this)); - - _services = new Lazy(() => - new Services(UU.ProcessorServices(proc, target), marsh, this, false, false)); + _comp = new Lazy(() => CreateCompute()); + _msg = new Lazy(() => CreateMessaging()); + _events = new Lazy(() => CreateEvents()); + _services = new Lazy(() => CreateServices()); } /** */ @@ -202,6 +202,14 @@ public ICompute GetCompute() return _comp.Value; } + /// + /// Creates the compute. + /// + private ICompute CreateCompute() + { + return new Compute(new ComputeImpl(DoOutOpObject(OpGetCompute), Marshaller, this, false)); + } + /** */ public IClusterGroup ForNodes(IEnumerable nodes) { @@ -257,7 +265,7 @@ public IClusterGroup ForPredicate(Func p) { var newPred = _pred == null ? p : node => _pred(node) && p(node); - return new ClusterGroupImpl(_proc, Target, Marshaller, _ignite, newPred); + return new ClusterGroupImpl(Target, _ignite, newPred); } /** */ @@ -413,18 +421,42 @@ public IMessaging GetMessaging() return _msg.Value; } + /// + /// Creates the messaging. + /// + private IMessaging CreateMessaging() + { + return new Messaging(DoOutOpObject(OpGetMessaging), Marshaller, this); + } + /** */ public IEvents GetEvents() { return _events.Value; } + /// + /// Creates the events. + /// + private IEvents CreateEvents() + { + return new Events(DoOutOpObject(OpGetEvents), Marshaller, this); + } + /** */ public IServices GetServices() { return _services.Value; } + /// + /// Creates the services. + /// + private IServices CreateServices() + { + return new Services(DoOutOpObject(OpGetServices), Marshaller, this, false, false); + } + /// /// Pings a remote node. /// @@ -635,7 +667,7 @@ public IPersistentStoreMetrics GetPersistentStoreMetrics() /// New cluster group. private IClusterGroup GetClusterGroup(IUnmanagedTarget prj) { - return new ClusterGroupImpl(_proc, prj, Marshaller, _ignite, _pred); + return new ClusterGroupImpl(prj, _ignite, _pred); } /// diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeImpl.cs index 7a028cd315522..cace7b2211770 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeImpl.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeImpl.cs @@ -186,10 +186,10 @@ public Future ExecuteJavaTaskAsync(string taskName, obje long ptr = Marshaller.Ignite.HandleRegistry.Allocate(holder); - var futTarget = DoOutOpObject(OpExecNative, w => + var futTarget = DoOutOpObject(OpExecNative, (IBinaryStream s) => { - w.WriteLong(ptr); - w.WriteLong(_prj.TopologyVersion); + s.WriteLong(ptr); + s.WriteLong(_prj.TopologyVersion); }); var future = holder.Future; diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerImpl.cs index b9e3030e90c4c..96e58d4cccc35 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerImpl.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerImpl.cs @@ -547,8 +547,7 @@ public void Close(bool cancel) return result; } - return new DataStreamerImpl(UU.ProcessorDataStreamer(Marshaller.Ignite.InteropProcessor, - _cacheName, true), Marshaller, _cacheName, true); + return Marshaller.Ignite.GetDataStreamer(_cacheName, true); } /** */ diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/StreamReceiverHolder.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/StreamReceiverHolder.cs index 953ddb6d43f47..c91334d1dce73 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/StreamReceiverHolder.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/StreamReceiverHolder.cs @@ -138,7 +138,7 @@ public void Receive(Ignite grid, IUnmanagedTarget cache, IBinaryStream stream, b for (var i = 0; i < size; i++) entries.Add(new CacheEntry(reader.ReadObject(), reader.ReadObject())); - receiver.Receive(grid.Cache(cache, keepBinary), entries); + receiver.Receive(grid.GetCache(cache, keepBinary), entries); } } } \ No newline at end of file diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs index 205f6e2ba3440..715776e2c2beb 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs @@ -42,6 +42,7 @@ namespace Apache.Ignite.Core.Impl using Apache.Ignite.Core.Impl.Plugin; using Apache.Ignite.Core.Impl.Transactions; using Apache.Ignite.Core.Impl.Unmanaged; + using Apache.Ignite.Core.Interop; using Apache.Ignite.Core.Lifecycle; using Apache.Ignite.Core.Log; using Apache.Ignite.Core.Messaging; @@ -53,8 +54,37 @@ namespace Apache.Ignite.Core.Impl /// /// Native Ignite wrapper. /// - internal class Ignite : IIgnite, ICluster + internal class Ignite : PlatformTarget, IIgnite, ICluster { + /// + /// Operation codes for PlatformProcessorImpl calls. + /// + private enum Op + { + GetCache = 1, + CreateCache = 2, + GetOrCreateCache = 3, + CreateCacheFromConfig = 4, + GetOrCreateCacheFromConfig = 5, + DestroyCache = 6, + GetAffinity = 7, + GetDataStreamer = 8, + GetTransactions = 9, + GetClusterGroup = 10, + GetExtension = 11, + GetAtomicLong = 12, + GetAtomicReference = 13, + GetAtomicSequence = 14, + GetIgniteConfiguration = 15, + GetCacheNames = 16, + CreateNearCache = 17, + GetOrCreateNearCache = 18, + LoggerIsLevelEnabled = 19, + LoggerLog = 20, + GetBinaryProcessor = 21, + ReleaseStart = 22 + } + /** */ private readonly IgniteConfiguration _cfg; @@ -109,7 +139,7 @@ internal class Ignite : IIgnite, ICluster /// Lifecycle beans. /// Callbacks. public Ignite(IgniteConfiguration cfg, string name, IUnmanagedTarget proc, Marshaller marsh, - IList lifecycleHandlers, UnmanagedCallbacks cbs) + IList lifecycleHandlers, UnmanagedCallbacks cbs) : base(proc, marsh) { Debug.Assert(cfg != null); Debug.Assert(proc != null); @@ -126,17 +156,17 @@ internal class Ignite : IIgnite, ICluster marsh.Ignite = this; - _prj = new ClusterGroupImpl(proc, UU.ProcessorProjection(proc), marsh, this, null); + _prj = new ClusterGroupImpl(DoOutOpObject((int) Op.GetClusterGroup), this, null); _binary = new Binary.Binary(marsh); - _binaryProc = new BinaryProcessor(UU.ProcessorBinaryProcessor(proc), marsh); + _binaryProc = new BinaryProcessor(DoOutOpObject((int) Op.GetBinaryProcessor), marsh); cbs.Initialize(this); // Grid is not completely started here, can't initialize interop transactions right away. _transactions = new Lazy( - () => new TransactionsImpl(UU.ProcessorTransactions(proc), marsh, GetLocalNode().Id)); + () => new TransactionsImpl(DoOutOpObject((int) Op.GetTransactions), marsh, GetLocalNode().Id)); // Set reconnected task to completed state for convenience. _clientReconnectTaskCompletionSource.SetResult(false); @@ -207,7 +237,7 @@ public ICompute GetCompute() /** */ public IClusterGroup ForNodes(IEnumerable nodes) { - return ((IClusterGroup) _prj).ForNodes(nodes); + return _prj.ForNodes(nodes); } /** */ @@ -218,12 +248,6 @@ public IClusterGroup ForNodes(params IClusterNode[] nodes) /** */ public IClusterGroup ForNodeIds(IEnumerable ids) - { - return ((IClusterGroup) _prj).ForNodeIds(ids); - } - - /** */ - public IClusterGroup ForNodeIds(ICollection ids) { return _prj.ForNodeIds(ids); } @@ -389,7 +413,8 @@ internal void AfterNodeStop() { IgniteArgumentCheck.NotNull(name, "name"); - return Cache(UU.ProcessorCache(_proc, name)); + + return GetCache(DoOutOpObject((int) Op.GetCache, w => w.WriteString(name))); } /** */ @@ -397,7 +422,7 @@ internal void AfterNodeStop() { IgniteArgumentCheck.NotNull(name, "name"); - return Cache(UU.ProcessorGetOrCreateCache(_proc, name)); + return GetCache(DoOutOpObject((int) Op.GetOrCreateCache, w => w.WriteString(name))); } /** */ @@ -407,31 +432,10 @@ internal void AfterNodeStop() } /** */ - public ICache GetOrCreateCache(CacheConfiguration configuration, + public ICache GetOrCreateCache(CacheConfiguration configuration, NearCacheConfiguration nearConfiguration) { - IgniteArgumentCheck.NotNull(configuration, "configuration"); - IgniteArgumentCheck.NotNull(configuration.Name, "CacheConfiguration.Name"); - configuration.Validate(Logger); - - using (var stream = IgniteManager.Memory.Allocate().GetStream()) - { - var writer = BinaryUtils.Marshaller.StartMarshal(stream); - - configuration.Write(writer); - - if (nearConfiguration != null) - { - writer.WriteBoolean(true); - nearConfiguration.Write(writer); - } - else - writer.WriteBoolean(false); - - stream.SynchronizeOutput(); - - return Cache(UU.ProcessorGetOrCreateCache(_proc, stream.MemoryPointer)); - } + return GetOrCreateCache(configuration, nearConfiguration, Op.GetOrCreateCacheFromConfig); } /** */ @@ -439,7 +443,9 @@ internal void AfterNodeStop() { IgniteArgumentCheck.NotNull(name, "name"); - return Cache(UU.ProcessorCreateCache(_proc, name)); + var cacheTarget = DoOutOpObject((int) Op.CreateCache, w => w.WriteString(name)); + + return GetCache(cacheTarget); } /** */ @@ -451,30 +457,38 @@ internal void AfterNodeStop() /** */ public ICache CreateCache(CacheConfiguration configuration, NearCacheConfiguration nearConfiguration) + { + return GetOrCreateCache(configuration, nearConfiguration, Op.CreateCacheFromConfig); + } + + /// + /// Gets or creates the cache. + /// + private ICache GetOrCreateCache(CacheConfiguration configuration, + NearCacheConfiguration nearConfiguration, Op op) { IgniteArgumentCheck.NotNull(configuration, "configuration"); IgniteArgumentCheck.NotNull(configuration.Name, "CacheConfiguration.Name"); configuration.Validate(Logger); - using (var stream = IgniteManager.Memory.Allocate().GetStream()) + var cacheTarget = DoOutOpObject((int) op, s => { - // Use system marshaller: full footers, always unregistered mode. - var writer = BinaryUtils.Marshaller.StartMarshal(stream); + var w = BinaryUtils.Marshaller.StartMarshal(s); - configuration.Write(writer); + configuration.Write(w); if (nearConfiguration != null) { - writer.WriteBoolean(true); - nearConfiguration.Write(writer); + w.WriteBoolean(true); + nearConfiguration.Write(w); } else - writer.WriteBoolean(false); - - stream.SynchronizeOutput(); + { + w.WriteBoolean(false); + } + }); - return Cache(UU.ProcessorCreateCache(_proc, stream.MemoryPointer)); - } + return GetCache(cacheTarget); } /** */ @@ -482,7 +496,7 @@ public void DestroyCache(string name) { IgniteArgumentCheck.NotNull(name, "name"); - UU.ProcessorDestroyCache(_proc, name); + DoOutOp((int) Op.DestroyCache, w => w.WriteString(name)); } /// @@ -493,7 +507,7 @@ public void DestroyCache(string name) /// /// New instance of cache wrapping specified native cache. /// - public ICache Cache(IUnmanagedTarget nativeCache, bool keepBinary = false) + public ICache GetCache(IUnmanagedTarget nativeCache, bool keepBinary = false) { return new CacheImpl(this, nativeCache, _marsh, false, keepBinary, false, false); } @@ -541,8 +555,21 @@ public Task ClientReconnectTask { IgniteArgumentCheck.NotNull(cacheName, "cacheName"); - return new DataStreamerImpl(UU.ProcessorDataStreamer(_proc, cacheName, false), - _marsh, cacheName, false); + return GetDataStreamer(cacheName, false); + } + + /// + /// Gets the data streamer. + /// + internal IDataStreamer GetDataStreamer(string cacheName, bool keepBinary) + { + var streamerTarget = DoOutOpObject((int) Op.GetDataStreamer, w => + { + w.WriteString(cacheName); + w.WriteBoolean(keepBinary); + }); + + return new DataStreamerImpl(streamerTarget, _marsh, cacheName, keepBinary); } /** */ @@ -556,11 +583,12 @@ public ICacheAffinity GetAffinity(string cacheName) { IgniteArgumentCheck.NotNull(cacheName, "cacheName"); - return new CacheAffinityImpl(UU.ProcessorAffinity(_proc, cacheName), _marsh, false, this); + var aff = DoOutOpObject((int) Op.GetAffinity, w => w.WriteString(cacheName)); + + return new CacheAffinityImpl(aff, _marsh, false, this); } /** */ - public ITransactions GetTransactions() { return _transactions.Value; @@ -589,7 +617,12 @@ public IAtomicLong GetAtomicLong(string name, long initialValue, bool create) { IgniteArgumentCheck.NotNullOrEmpty(name, "name"); - var nativeLong = UU.ProcessorAtomicLong(_proc, name, initialValue, create); + var nativeLong = DoOutOpObject((int) Op.GetAtomicLong, w => + { + w.WriteString(name); + w.WriteLong(initialValue); + w.WriteBoolean(create); + }); if (nativeLong == null) return null; @@ -602,7 +635,12 @@ public IAtomicSequence GetAtomicSequence(string name, long initialValue, bool cr { IgniteArgumentCheck.NotNullOrEmpty(name, "name"); - var nativeSeq = UU.ProcessorAtomicSequence(_proc, name, initialValue, create); + var nativeSeq = DoOutOpObject((int) Op.GetAtomicSequence, w => + { + w.WriteString(name); + w.WriteLong(initialValue); + w.WriteBoolean(create); + }); if (nativeSeq == null) return null; @@ -615,81 +653,47 @@ public IAtomicReference GetAtomicReference(string name, T initialValue, bo { IgniteArgumentCheck.NotNullOrEmpty(name, "name"); - var refTarget = GetAtomicReferenceUnmanaged(name, initialValue, create); - - return refTarget == null ? null : new AtomicReference(refTarget, Marshaller, name); - } - - /// - /// Gets the unmanaged atomic reference. - /// - /// The name. - /// The initial value. - /// Create flag. - /// Unmanaged atomic reference, or null. - private IUnmanagedTarget GetAtomicReferenceUnmanaged(string name, T initialValue, bool create) - { - IgniteArgumentCheck.NotNullOrEmpty(name, "name"); - - // Do not allocate memory when default is not used. - if (!create) - return UU.ProcessorAtomicReference(_proc, name, 0, false); - - using (var stream = IgniteManager.Memory.Allocate().GetStream()) + var refTarget = DoOutOpObject((int) Op.GetAtomicReference, w => { - var writer = Marshaller.StartMarshal(stream); - - writer.Write(initialValue); - - Marshaller.FinishMarshal(writer); + w.WriteString(name); + w.WriteObject(initialValue); + w.WriteBoolean(create); + }); - var memPtr = stream.SynchronizeOutput(); - - return UU.ProcessorAtomicReference(_proc, name, memPtr, true); - } + return refTarget == null ? null : new AtomicReference(refTarget, Marshaller, name); } /** */ public IgniteConfiguration GetConfiguration() { - using (var stream = IgniteManager.Memory.Allocate(1024).GetStream()) - { - UU.ProcessorGetIgniteConfiguration(_proc, stream.MemoryPointer); - - stream.SynchronizeInput(); - - return new IgniteConfiguration(BinaryUtils.Marshaller.StartUnmarshal(stream), _cfg); - } + return DoInOp((int) Op.GetIgniteConfiguration, + s => new IgniteConfiguration(BinaryUtils.Marshaller.StartUnmarshal(s), _cfg)); } /** */ public ICache CreateNearCache(string name, NearCacheConfiguration configuration) { - return GetOrCreateNearCache0(name, configuration, UU.ProcessorCreateNearCache); + return GetOrCreateNearCache0(name, configuration, Op.CreateNearCache); } /** */ public ICache GetOrCreateNearCache(string name, NearCacheConfiguration configuration) { - return GetOrCreateNearCache0(name, configuration, UU.ProcessorGetOrCreateNearCache); + return GetOrCreateNearCache0(name, configuration, Op.GetOrCreateNearCache); } /** */ public ICollection GetCacheNames() { - using (var stream = IgniteManager.Memory.Allocate(1024).GetStream()) + return OutStream((int) Op.GetCacheNames, r => { - UU.ProcessorGetCacheNames(_proc, stream.MemoryPointer); - stream.SynchronizeInput(); - - var reader = _marsh.StartUnmarshal(stream); - var res = new string[stream.ReadInt()]; + var res = new string[r.ReadInt()]; - for (int i = 0; i < res.Length; i++) - res[i] = reader.ReadString(); + for (var i = 0; i < res.Length; i++) + res[i] = r.ReadString(); - return res; - } + return (ICollection) res; + }); } /** */ @@ -768,20 +772,17 @@ public IPersistentStoreMetrics GetPersistentStoreMetrics() /// Gets or creates near cache. /// private ICache GetOrCreateNearCache0(string name, NearCacheConfiguration configuration, - Func func) + Op op) { IgniteArgumentCheck.NotNull(configuration, "configuration"); - using (var stream = IgniteManager.Memory.Allocate().GetStream()) + var cacheTarget = DoOutOpObject((int) op, w => { - var writer = BinaryUtils.Marshaller.StartMarshal(stream); + w.WriteString(name); + configuration.Write(w); + }); - configuration.Write(writer); - - stream.SynchronizeOutput(); - - return Cache(func(_proc, name, stream.MemoryPointer)); - } + return GetCache(cacheTarget); } /// @@ -793,14 +794,6 @@ internal ClusterGroupImpl ClusterGroup get { return _prj; } } - /// - /// Marshaller. - /// - internal Marshaller Marshaller - { - get { return _marsh; } - } - /// /// Gets the binary processor. /// @@ -892,5 +885,43 @@ internal PluginProcessor PluginProcessor { get { return _pluginProcessor; } } + + /// + /// Notify processor that it is safe to use. + /// + internal void ProcessorReleaseStart() + { + InLongOutLong((int) Op.ReleaseStart, 0); + } + + /// + /// Checks whether log level is enabled in Java logger. + /// + internal bool LoggerIsLevelEnabled(LogLevel logLevel) + { + return InLongOutLong((int) Op.LoggerIsLevelEnabled, (long) logLevel) == True; + } + + /// + /// Logs to the Java logger. + /// + internal void LoggerLog(LogLevel level, string msg, string category, string err) + { + InStreamOutLong((int) Op.LoggerLog, w => + { + w.WriteInt((int) level); + w.WriteString(msg); + w.WriteString(category); + w.WriteString(err); + }); + } + + /// + /// Gets the platform plugin extension. + /// + internal IPlatformTarget GetExtension(int id) + { + return InStreamOutObject((int) Op.GetExtension, w => w.WriteInt(id)); + } } } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Log/JavaLogger.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Log/JavaLogger.cs index 23e7a3733ca6c..2e47fe8942bad 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Log/JavaLogger.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Log/JavaLogger.cs @@ -21,7 +21,6 @@ namespace Apache.Ignite.Core.Impl.Log using System.Collections.Generic; using System.Diagnostics; using System.Linq; - using Apache.Ignite.Core.Impl.Unmanaged; using Apache.Ignite.Core.Log; /// @@ -30,7 +29,7 @@ namespace Apache.Ignite.Core.Impl.Log internal class JavaLogger : ILogger { /** */ - private IUnmanagedTarget _proc; + private Ignite _ignite; /** */ private readonly List _enabledLevels = new List(5); @@ -45,19 +44,19 @@ internal class JavaLogger : ILogger /// /// Sets the processor. /// - /// The proc. - public void SetProcessor(IUnmanagedTarget proc) + /// The proc. + public void SetIgnite(Ignite ignite) { - Debug.Assert(proc != null); + Debug.Assert(ignite != null); lock (_syncRoot) { - _proc = proc; + _ignite = ignite; // Preload enabled levels. _enabledLevels.AddRange( - new[] { LogLevel.Trace, LogLevel.Debug, LogLevel.Info, LogLevel.Warn, LogLevel.Error } - .Where(x => UnmanagedUtils.ProcessorLoggerIsLevelEnabled(proc, (int)x))); + new[] {LogLevel.Trace, LogLevel.Debug, LogLevel.Info, LogLevel.Warn, LogLevel.Error} + .Where(x => ignite.LoggerIsLevelEnabled(x))); foreach (var log in _pendingLogs) { @@ -82,7 +81,7 @@ public void SetProcessor(IUnmanagedTarget proc) var msg = args == null ? message : string.Format(formatProvider, message, args); var err = ex != null ? ex.ToString() : null; - if (_proc != null) + if (_ignite != null) Log(level, msg, category, err); else _pendingLogs.Add(Tuple.Create(level, msg, category, err)); @@ -94,7 +93,7 @@ public bool IsEnabled(LogLevel level) { lock (_syncRoot) { - return _proc == null || _enabledLevels.Contains(level); + return _ignite == null || _enabledLevels.Contains(level); } } @@ -104,7 +103,9 @@ public bool IsEnabled(LogLevel level) private void Log(LogLevel level, string msg, string category, string err) { if (IsEnabled(level)) - UnmanagedUtils.ProcessorLoggerLog(_proc, (int)level, msg, category, err); + { + _ignite.LoggerLog(level, msg, category, err); + } } } } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/PlatformTarget.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/PlatformTarget.cs index 8dd8eaf7fbfee..474af0e37b961 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/PlatformTarget.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/PlatformTarget.cs @@ -300,7 +300,7 @@ protected long DoOutOp(int type, Action action) /// /// Operation type. /// Action to be performed on the stream. - /// + /// Resulting object. protected IUnmanagedTarget DoOutOpObject(int type, Action action) { using (var stream = IgniteManager.Memory.Allocate().GetStream()) @@ -315,6 +315,22 @@ protected IUnmanagedTarget DoOutOpObject(int type, Action action) } } + /// + /// Perform out operation. + /// + /// Operation type. + /// Action to be performed on the stream. + /// Resulting object. + protected IUnmanagedTarget DoOutOpObject(int type, Action action) + { + using (var stream = IgniteManager.Memory.Allocate().GetStream()) + { + action(stream); + + return UU.TargetInStreamOutObject(_target, type, stream.SynchronizeOutput()); + } + } + /// /// Perform out operation. /// diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Plugin/PluginContext.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Plugin/PluginContext.cs index cc20cb31d538e..eac7556a668c8 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Plugin/PluginContext.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Plugin/PluginContext.cs @@ -20,7 +20,6 @@ namespace Apache.Ignite.Core.Impl.Plugin using Apache.Ignite.Core.Common; using Apache.Ignite.Core.Impl.Common; using Apache.Ignite.Core.Impl.Resource; - using Apache.Ignite.Core.Impl.Unmanaged; using Apache.Ignite.Core.Interop; using Apache.Ignite.Core.Plugin; @@ -65,11 +64,7 @@ public T PluginConfiguration /** */ public IPlatformTarget GetExtension(int id) { - var ignite = _pluginProcessor.Ignite; - - var ext = UnmanagedUtils.ProcessorExtension(ignite.InteropProcessor, id); - - return new PlatformTarget(ext, ignite.Marshaller); + return _pluginProcessor.Ignite.GetExtension(id); } /** */ diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/IgniteJniNativeMethods.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/IgniteJniNativeMethods.cs index 289589fb9091a..1720a79ded668 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/IgniteJniNativeMethods.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/IgniteJniNativeMethods.cs @@ -30,105 +30,16 @@ internal static unsafe class IgniteJniNativeMethods public static extern int Reallocate(long memPtr, int cap); [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteIgnitionStart")] - public static extern void* IgnitionStart(void* ctx, sbyte* cfgPath, sbyte* gridName, int factoryId, + public static extern void IgnitionStart(void* ctx, sbyte* cfgPath, sbyte* gridName, int factoryId, long dataPtr); [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteIgnitionStop")] [return: MarshalAs(UnmanagedType.U1)] public static extern bool IgnitionStop(void* ctx, sbyte* gridName, [MarshalAs(UnmanagedType.U1)] bool cancel); - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteIgnitionStopAll")] - public static extern void IgnitionStopAll(void* ctx, [MarshalAs(UnmanagedType.U1)] bool cancel); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorReleaseStart")] - public static extern void ProcessorReleaseStart(void* ctx, void* obj); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorProjection")] - public static extern void* ProcessorProjection(void* ctx, void* obj); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorCache")] - public static extern void* ProcessorCache(void* ctx, void* obj, sbyte* name); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorCreateCache")] - public static extern void* ProcessorCreateCache(void* ctx, void* obj, sbyte* name); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorCreateCacheFromConfig")] - public static extern void* ProcessorCreateCacheFromConfig(void* ctx, void* obj, long memPtr); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorGetOrCreateCache")] - public static extern void* ProcessorGetOrCreateCache(void* ctx, void* obj, sbyte* name); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorGetOrCreateCacheFromConfig")] - public static extern void* ProcessorGetOrCreateCacheFromConfig(void* ctx, void* obj, long memPtr); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorCreateNearCache")] - public static extern void* ProcessorCreateNearCache(void* ctx, void* obj, sbyte* name, long memPtr); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorGetOrCreateNearCache")] - public static extern void* ProcessorGetOrCreateNearCache(void* ctx, void* obj, sbyte* name, long memPtr); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorDestroyCache")] - public static extern void ProcessorDestroyCache(void* ctx, void* obj, sbyte* name); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorAffinity")] - public static extern void* ProcessorAffinity(void* ctx, void* obj, sbyte* name); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorDataStreamer")] - public static extern void* ProcessorDataStreamer(void* ctx, void* obj, sbyte* name, - [MarshalAs(UnmanagedType.U1)] bool keepBinary); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorTransactions")] - public static extern void* ProcessorTransactions(void* ctx, void* obj); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorCompute")] - public static extern void* ProcessorCompute(void* ctx, void* obj, void* prj); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorMessage")] - public static extern void* ProcessorMessage(void* ctx, void* obj, void* prj); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorEvents")] - public static extern void* ProcessorEvents(void* ctx, void* obj, void* prj); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorServices")] - public static extern void* ProcessorServices(void* ctx, void* obj, void* prj); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorExtensions")] - public static extern void* ProcessorExtensions(void* ctx, void* obj); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorExtension")] - public static extern void* ProcessorExtension(void* ctx, void* obj, int id); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorAtomicLong")] - public static extern void* ProcessorAtomicLong(void* ctx, void* obj, sbyte* name, long initVal, - [MarshalAs(UnmanagedType.U1)] bool create); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorAtomicSequence")] - public static extern void* ProcessorAtomicSequence(void* ctx, void* obj, sbyte* name, long initVal, - [MarshalAs(UnmanagedType.U1)] bool create); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorAtomicReference")] - public static extern void* ProcessorAtomicReference(void* ctx, void* obj, sbyte* name, long memPtr, - [MarshalAs(UnmanagedType.U1)] bool create); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorGetIgniteConfiguration")] - public static extern void ProcessorGetIgniteConfiguration(void* ctx, void* obj, long memPtr); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorGetCacheNames")] - public static extern void ProcessorGetCacheNames(void* ctx, void* obj, long memPtr); - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteTargetInLongOutLong")] public static extern long TargetInLongOutLong(void* ctx, void* target, int opType, long val); - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorLoggerIsLevelEnabled")] - [return: MarshalAs(UnmanagedType.U1)] - public static extern bool ProcessorLoggerIsLevelEnabled(void* ctx, void* obj, int level); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorLoggerLog")] - public static extern void ProcessorLoggerLog(void* ctx, void* obj, int level, sbyte* messsage, sbyte* category, sbyte* errorInfo); - - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteProcessorBinaryProcessor")] - public static extern void* ProcessorBinaryProcessor(void* ctx, void* obj); - [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteTargetInStreamOutLong")] public static extern long TargetInStreamOutLong(void* ctx, void* target, int opType, long memPtr); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedUtils.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedUtils.cs index f76bbaca5cfec..a38cf2f9fa6cd 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedUtils.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedUtils.cs @@ -93,12 +93,9 @@ internal static void Initialize() try { - // OnStart receives the same InteropProcessor as here (just as another GlobalRef) and stores it. - // Release current reference immediately. - void* res = JNI.IgnitionStart(ctx.NativeContext, cfgPath0, gridName0, InteropFactoryId, + // OnStart receives InteropProcessor referece and stores it. + JNI.IgnitionStart(ctx.NativeContext, cfgPath0, gridName0, InteropFactoryId, mem.SynchronizeOutput()); - - JNI.Release(res); } finally { @@ -122,304 +119,6 @@ internal static bool IgnitionStop(void* ctx, string gridName, bool cancel) } } - internal static void IgnitionStopAll(void* ctx, bool cancel) - { - JNI.IgnitionStopAll(ctx, cancel); - } - - internal static void ProcessorReleaseStart(IUnmanagedTarget target) - { - JNI.ProcessorReleaseStart(target.Context, target.Target); - } - - internal static IUnmanagedTarget ProcessorProjection(IUnmanagedTarget target) - { - void* res = JNI.ProcessorProjection(target.Context, target.Target); - - return target.ChangeTarget(res); - } - - internal static IUnmanagedTarget ProcessorCache(IUnmanagedTarget target, string name) - { - sbyte* name0 = IgniteUtils.StringToUtf8Unmanaged(name); - - try - { - void* res = JNI.ProcessorCache(target.Context, target.Target, name0); - - return target.ChangeTarget(res); - } - finally - { - Marshal.FreeHGlobal(new IntPtr(name0)); - } - } - - internal static IUnmanagedTarget ProcessorCreateCache(IUnmanagedTarget target, string name) - { - sbyte* name0 = IgniteUtils.StringToUtf8Unmanaged(name); - - try - { - void* res = JNI.ProcessorCreateCache(target.Context, target.Target, name0); - - return target.ChangeTarget(res); - } - finally - { - Marshal.FreeHGlobal(new IntPtr(name0)); - } - } - - internal static IUnmanagedTarget ProcessorCreateCache(IUnmanagedTarget target, long memPtr) - { - void* res = JNI.ProcessorCreateCacheFromConfig(target.Context, target.Target, memPtr); - - return target.ChangeTarget(res); - } - - internal static IUnmanagedTarget ProcessorGetOrCreateCache(IUnmanagedTarget target, string name) - { - sbyte* name0 = IgniteUtils.StringToUtf8Unmanaged(name); - - try - { - void* res = JNI.ProcessorGetOrCreateCache(target.Context, target.Target, name0); - - return target.ChangeTarget(res); - } - finally - { - Marshal.FreeHGlobal(new IntPtr(name0)); - } - } - - internal static IUnmanagedTarget ProcessorGetOrCreateCache(IUnmanagedTarget target, long memPtr) - { - void* res = JNI.ProcessorGetOrCreateCacheFromConfig(target.Context, target.Target, memPtr); - - return target.ChangeTarget(res); - } - - internal static IUnmanagedTarget ProcessorCreateNearCache(IUnmanagedTarget target, string name, long memPtr) - { - sbyte* name0 = IgniteUtils.StringToUtf8Unmanaged(name); - - try - { - void* res = JNI.ProcessorCreateNearCache(target.Context, target.Target, name0, memPtr); - - return target.ChangeTarget(res); - } - finally - { - Marshal.FreeHGlobal(new IntPtr(name0)); - } - } - - internal static IUnmanagedTarget ProcessorGetOrCreateNearCache(IUnmanagedTarget target, string name, long memPtr) - { - sbyte* name0 = IgniteUtils.StringToUtf8Unmanaged(name); - - try - { - void* res = JNI.ProcessorGetOrCreateNearCache(target.Context, target.Target, name0, memPtr); - - return target.ChangeTarget(res); - } - finally - { - Marshal.FreeHGlobal(new IntPtr(name0)); - } - } - - internal static void ProcessorDestroyCache(IUnmanagedTarget target, string name) - { - sbyte* name0 = IgniteUtils.StringToUtf8Unmanaged(name); - - try - { - JNI.ProcessorDestroyCache(target.Context, target.Target, name0); - } - finally - { - Marshal.FreeHGlobal(new IntPtr(name0)); - } - } - - internal static IUnmanagedTarget ProcessorAffinity(IUnmanagedTarget target, string name) - { - sbyte* name0 = IgniteUtils.StringToUtf8Unmanaged(name); - - try - { - void* res = JNI.ProcessorAffinity(target.Context, target.Target, name0); - - return target.ChangeTarget(res); - } - finally - { - Marshal.FreeHGlobal(new IntPtr(name0)); - } - } - - internal static IUnmanagedTarget ProcessorDataStreamer(IUnmanagedTarget target, string name, bool keepBinary) - { - sbyte* name0 = IgniteUtils.StringToUtf8Unmanaged(name); - - try - { - void* res = JNI.ProcessorDataStreamer(target.Context, target.Target, name0, keepBinary); - - return target.ChangeTarget(res); - } - finally - { - Marshal.FreeHGlobal(new IntPtr(name0)); - } - } - - internal static IUnmanagedTarget ProcessorTransactions(IUnmanagedTarget target) - { - void* res = JNI.ProcessorTransactions(target.Context, target.Target); - - return target.ChangeTarget(res); - } - - internal static IUnmanagedTarget ProcessorCompute(IUnmanagedTarget target, IUnmanagedTarget prj) - { - void* res = JNI.ProcessorCompute(target.Context, target.Target, prj.Target); - - return target.ChangeTarget(res); - } - - internal static IUnmanagedTarget ProcessorMessage(IUnmanagedTarget target, IUnmanagedTarget prj) - { - void* res = JNI.ProcessorMessage(target.Context, target.Target, prj.Target); - - return target.ChangeTarget(res); - } - - internal static IUnmanagedTarget ProcessorEvents(IUnmanagedTarget target, IUnmanagedTarget prj) - { - void* res = JNI.ProcessorEvents(target.Context, target.Target, prj.Target); - - return target.ChangeTarget(res); - } - - internal static IUnmanagedTarget ProcessorServices(IUnmanagedTarget target, IUnmanagedTarget prj) - { - void* res = JNI.ProcessorServices(target.Context, target.Target, prj.Target); - - return target.ChangeTarget(res); - } - - internal static IUnmanagedTarget ProcessorExtensions(IUnmanagedTarget target) - { - void* res = JNI.ProcessorExtensions(target.Context, target.Target); - - return target.ChangeTarget(res); - } - - internal static IUnmanagedTarget ProcessorExtension(IUnmanagedTarget target, int id) - { - void* res = JNI.ProcessorExtension(target.Context, target.Target, id); - - return target.ChangeTarget(res); - } - - internal static IUnmanagedTarget ProcessorAtomicLong(IUnmanagedTarget target, string name, long initialValue, - bool create) - { - var name0 = IgniteUtils.StringToUtf8Unmanaged(name); - - try - { - var res = JNI.ProcessorAtomicLong(target.Context, target.Target, name0, initialValue, create); - - return res == null ? null : target.ChangeTarget(res); - } - finally - { - Marshal.FreeHGlobal(new IntPtr(name0)); - } - } - - internal static IUnmanagedTarget ProcessorAtomicSequence(IUnmanagedTarget target, string name, long initialValue, - bool create) - { - var name0 = IgniteUtils.StringToUtf8Unmanaged(name); - - try - { - var res = JNI.ProcessorAtomicSequence(target.Context, target.Target, name0, initialValue, create); - - return res == null ? null : target.ChangeTarget(res); - } - finally - { - Marshal.FreeHGlobal(new IntPtr(name0)); - } - } - - internal static IUnmanagedTarget ProcessorAtomicReference(IUnmanagedTarget target, string name, long memPtr, - bool create) - { - var name0 = IgniteUtils.StringToUtf8Unmanaged(name); - - try - { - var res = JNI.ProcessorAtomicReference(target.Context, target.Target, name0, memPtr, create); - - return res == null ? null : target.ChangeTarget(res); - } - finally - { - Marshal.FreeHGlobal(new IntPtr(name0)); - } - } - - internal static void ProcessorGetIgniteConfiguration(IUnmanagedTarget target, long memPtr) - { - JNI.ProcessorGetIgniteConfiguration(target.Context, target.Target, memPtr); - } - - internal static void ProcessorGetCacheNames(IUnmanagedTarget target, long memPtr) - { - JNI.ProcessorGetCacheNames(target.Context, target.Target, memPtr); - } - - internal static bool ProcessorLoggerIsLevelEnabled(IUnmanagedTarget target, int level) - { - return JNI.ProcessorLoggerIsLevelEnabled(target.Context, target.Target, level); - } - - internal static void ProcessorLoggerLog(IUnmanagedTarget target, int level, string message, string category, - string errorInfo) - { - var message0 = IgniteUtils.StringToUtf8Unmanaged(message); - var category0 = IgniteUtils.StringToUtf8Unmanaged(category); - var errorInfo0 = IgniteUtils.StringToUtf8Unmanaged(errorInfo); - - try - { - JNI.ProcessorLoggerLog(target.Context, target.Target, level, message0, category0, errorInfo0); - } - finally - { - Marshal.FreeHGlobal(new IntPtr(message0)); - Marshal.FreeHGlobal(new IntPtr(category0)); - Marshal.FreeHGlobal(new IntPtr(errorInfo0)); - } - } - - internal static IUnmanagedTarget ProcessorBinaryProcessor(IUnmanagedTarget target) - { - void* res = JNI.ProcessorBinaryProcessor(target.Context, target.Target); - - return target.ChangeTarget(res); - } - #endregion #region NATIVE METHODS: TARGET @@ -443,6 +142,9 @@ internal static IUnmanagedTarget TargetInStreamOutObject(IUnmanagedTarget target { void* res = JNI.TargetInStreamOutObject(target.Context, target.Target, opType, inMemPtr); + if (res == null) + return null; + return target.ChangeTarget(res); } From 131847203d55f4f3907a7ff62d53ff69345fa605 Mon Sep 17 00:00:00 2001 From: Yury Babak Date: Tue, 25 Jul 2017 15:22:34 +0300 Subject: [PATCH 023/547] IGNITE-5802: Fix all wrong TODO comments in ML component This closes #2340 --- .../ml/clustering/KMeansDistributedClusterer.java | 6 +++--- .../ignite/ml/clustering/KMeansLocalClusterer.java | 2 +- .../java/org/apache/ignite/ml/math/MathUtils.java | 1 + .../java/org/apache/ignite/ml/math/VectorUtils.java | 7 +++++-- .../ml/math/decompositions/LUDecomposition.java | 2 +- .../ml/math/decompositions/QRDecomposition.java | 2 +- .../math/exceptions/MathIllegalNumberException.java | 8 ++++---- .../org/apache/ignite/ml/math/impls/CacheUtils.java | 4 ++-- .../ignite/ml/math/impls/matrix/AbstractMatrix.java | 2 -- .../storage/matrix/DenseOffHeapMatrixStorage.java | 2 +- .../vector/DenseLocalOffHeapVectorStorage.java | 2 +- .../vector/SparseLocalOffHeapVectorStorage.java | 2 +- .../vector/SparseLocalOnHeapVectorStorage.java | 4 ++-- .../ignite/ml/math/impls/vector/CacheVector.java | 4 ++-- .../ml/math/impls/vector/MapWrapperVector.java | 1 - .../ml/math/impls/vector/SparseLocalVector.java | 6 +++--- .../ignite/ml/math/impls/vector/VectorView.java | 2 +- .../org/apache/ignite/ml/math/util/MatrixUtil.java | 4 ++-- .../AbstractMultipleLinearRegression.java | 4 ++-- .../ml/regressions/OLSMultipleLinearRegression.java | 2 +- .../test/java/org/apache/ignite/ml/TestUtils.java | 2 +- .../ignite/ml/math/benchmark/MathBenchmark.java | 2 +- .../ml/math/benchmark/VectorBenchmarkTest.java | 2 +- .../math/decompositions/EigenDecompositionTest.java | 2 +- .../ml/math/impls/matrix/MatrixAttributeTest.java | 2 -- .../matrix/MatrixStorageImplementationTest.java | 2 +- .../ml/math/impls/vector/AbstractVectorTest.java | 4 ++-- .../vector/PivotedVectorViewConstructorTest.java | 2 +- .../ml/math/impls/vector/VectorAttributesTest.java | 2 +- .../ml/math/impls/vector/VectorFoldMapTest.java | 2 +- .../math/impls/vector/VectorImplementationsTest.java | 12 ++++++------ .../ml/math/impls/vector/VectorIterableTest.java | 2 +- .../ml/math/impls/vector/VectorToMatrixTest.java | 4 ++-- .../regressions/OLSMultipleLinearRegressionTest.java | 4 ++-- 34 files changed, 55 insertions(+), 56 deletions(-) diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java index 5f702d2f38078..2ef61ad4c5b39 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java @@ -80,7 +80,7 @@ public KMeansDistributedClusterer(DistanceMeasure measure, int initSteps, int ma MathIllegalArgumentException, ConvergenceException { SparseDistributedMatrix pointsCp = (SparseDistributedMatrix)points.like(points.rowSize(), points.columnSize()); - // TODO: this copy is very ineffective, just for POC. Immutability of data should be guaranteed by other methods + // TODO: IGNITE-5825, this copy is very ineffective, just for POC. Immutability of data should be guaranteed by other methods // such as logical locks for example. pointsCp.assign(points); @@ -117,10 +117,10 @@ public KMeansDistributedClusterer(DistanceMeasure measure, int initSteps, int ma /** Initialize cluster centers. */ private Vector[] initClusterCenters(SparseDistributedMatrix points, int k) { // Initialize empty centers and point costs. - int ptsCount = points.rowSize(); + int ptsCnt = points.rowSize(); // Initialize the first center to a random point. - Vector sample = localCopyOf(points.viewRow(rnd.nextInt(ptsCount))); + Vector sample = localCopyOf(points.viewRow(rnd.nextInt(ptsCnt))); List centers = new ArrayList<>(); List newCenters = new ArrayList<>(); diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansLocalClusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansLocalClusterer.java index 9f2aefbeaf69f..3d005b42f6d87 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansLocalClusterer.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansLocalClusterer.java @@ -92,7 +92,7 @@ public KMeansLocalClusterer(DistanceMeasure measure, int maxIterations, Long see } if (j == 0) - // TODO: Process this case more carefully + // TODO: IGNITE-5825, Process this case more carefully centers[i] = localCopyOf(points.viewRow(0)); else centers[i] = localCopyOf(points.viewRow(j - 1)); diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/MathUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/MathUtils.java index c2164ab9b778d..c7b33c3c3626a 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/MathUtils.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/MathUtils.java @@ -23,6 +23,7 @@ * Miscellaneous utility functions. */ public final class MathUtils { + /** */ public static void checkNotNull(Object o) throws NullArgumentException { if (o == null) diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java index 061d5ae163f1c..6bb0276f6ce3b 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java @@ -21,6 +21,9 @@ import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector; import org.apache.ignite.ml.math.impls.vector.MapWrapperVector; +/** + * Some utils for {@link Vector}. + */ public class VectorUtils { /** Create new vector like given vector initialized by zeroes. */ public static Vector zeroesLike(Vector v) { @@ -33,7 +36,7 @@ public static DenseLocalOnHeapVector zeroes(int n) { } /** */ - public static Vector fromMap(Map value, boolean copy) { - return new MapWrapperVector(value); + public static Vector fromMap(Map val, boolean cp) { + return new MapWrapperVector(val); } } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/LUDecomposition.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/LUDecomposition.java index 4c388b360d933..df7b6cd1d2f4f 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/LUDecomposition.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/LUDecomposition.java @@ -35,7 +35,7 @@ * @see MathWorld * @see Wikipedia * - * TODO: Maybe we should make this class (and other decompositions) Externalizable. + * TODO: IGNITE-5828, Maybe we should make this class (and other decompositions) Externalizable. */ public class LUDecomposition implements Destroyable { /** Default bound to determine effective singularity in LU decomposition. */ diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/QRDecomposition.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/QRDecomposition.java index 5ffa57481f4f9..3d0bb5dc6b2dd 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/QRDecomposition.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/QRDecomposition.java @@ -227,7 +227,7 @@ public Vector solve(Vector vec) { * @throws SingularMatrixException if the matrix is singular and {@code raise} is {@code true}. */ private static boolean checkSingular(Matrix r, double min, boolean raise) { - // TODO: Not a very fast approach for distributed matrices. would be nice if we could independently check + // TODO: IGNITE-5828, Not a very fast approach for distributed matrices. would be nice if we could independently check // parts on different nodes for singularity and do fold with 'or'. final int len = r.columnSize(); diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalNumberException.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalNumberException.java index 2e7280b35ad00..b2abf6388ebcd 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalNumberException.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalNumberException.java @@ -28,7 +28,7 @@ public class MathIllegalNumberException extends MathIllegalArgumentException { private static final long serialVersionUID = -7447085893598031110L; /** Requested. */ - private final Number argument; + private final Number arg; /** * Construct an exception. @@ -39,13 +39,13 @@ public class MathIllegalNumberException extends MathIllegalArgumentException { */ protected MathIllegalNumberException(String msg, Number wrong, Object... arguments) { super(msg, wrong, arguments); - argument = wrong; + arg = wrong; } /** * @return the requested value. */ - public Number getArgument() { - return argument; + public Number getArg() { + return arg; } } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java index 369840b2bf2b7..2c519f09a80c9 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java @@ -30,6 +30,7 @@ import org.apache.ignite.cluster.ClusterGroup; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.processors.cache.CacheEntryImpl; +import org.apache.ignite.internal.util.typedef.internal.A; import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.lang.IgniteCallable; import org.apache.ignite.lang.IgnitePredicate; @@ -43,12 +44,11 @@ import org.apache.ignite.ml.math.functions.IgniteFunction; import org.apache.ignite.ml.math.impls.matrix.BlockEntry; import org.apache.ignite.ml.math.impls.storage.matrix.BlockMatrixKey; -import org.apache.ignite.internal.util.typedef.internal.A; /** * Distribution-related misc. support. * - * TODO: IGNITE-5102, fix sparse key filters + * TODO: IGNITE-5102, fix sparse key filters. */ public class CacheUtils { /** diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java index 3dc9b43a0ad89..647ebc6dd811b 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java @@ -44,8 +44,6 @@ * interface to minimize the effort required to implement it. * Subclasses may override some of the implemented methods if a more * specific or optimized implementation is desirable. - * - * TODO: add row/column optimization. */ public abstract class AbstractMatrix implements Matrix { // Stochastic sparsity analysis. diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/DenseOffHeapMatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/DenseOffHeapMatrixStorage.java index 62060d133a615..7405a4e080c9f 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/DenseOffHeapMatrixStorage.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/DenseOffHeapMatrixStorage.java @@ -33,7 +33,7 @@ public class DenseOffHeapMatrixStorage implements MatrixStorage { private int cols; /** */ private transient long ptr; - //TODO: temp solution. + //TODO: IGNITE-5535, temp solution. /** */ private int ptrInitHash; diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/DenseLocalOffHeapVectorStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/DenseLocalOffHeapVectorStorage.java index a9965cc62edf0..71b7793b3b112 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/DenseLocalOffHeapVectorStorage.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/DenseLocalOffHeapVectorStorage.java @@ -33,7 +33,7 @@ public class DenseLocalOffHeapVectorStorage implements VectorStorage { /** */ private transient long ptr; - //TODO: temp solution. + //TODO: IGNITE-5535, temp solution. /** */ private int ptrInitHash; diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOffHeapVectorStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOffHeapVectorStorage.java index f6148c8a78034..a858364ee5880 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOffHeapVectorStorage.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOffHeapVectorStorage.java @@ -76,7 +76,7 @@ else if (gridOffHeapMap.contains(hash(i), intToByteArray(i))) /** {@inheritDoc} */ @Override public void writeExternal(ObjectOutput out) throws IOException { - throw new UnsupportedOperationException(); // TODO: add externalization support. + throw new UnsupportedOperationException(); // TODO: IGNITE-5801, add externalization support. } /** {@inheritDoc} */ diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java index 5145376630066..deef0109a7d99 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java @@ -47,7 +47,7 @@ public SparseLocalOnHeapVectorStorage() { } /** */ - public SparseLocalOnHeapVectorStorage(Map map, boolean copy) { + public SparseLocalOnHeapVectorStorage(Map map, boolean cp) { assert map.size() > 0; this.size = map.size(); @@ -59,7 +59,7 @@ else if (map instanceof Int2DoubleOpenHashMap) else acsMode = UNKNOWN_STORAGE_MODE; - if (copy) + if (cp) switch (acsMode) { case SEQUENTIAL_ACCESS_MODE: sto = new Int2DoubleRBTreeMap(map); diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/CacheVector.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/CacheVector.java index f78113ccfbad4..fec230fff69dd 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/CacheVector.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/CacheVector.java @@ -88,8 +88,8 @@ private Vector mapOverCache(IgniteFunction mapper) { /** {@inheritDoc} */ @Override public Vector map(IgniteBiFunction fun, double y) { - // TODO: provide cache-optimized implementation. - return super.map(fun, y); // TODO + // TODO: IGNTIE-5723, provide cache-optimized implementation. + return super.map(fun, y); } /** {@inheritDoc} */ diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/MapWrapperVector.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/MapWrapperVector.java index a093d0e68793e..83b40c19ae3fb 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/MapWrapperVector.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/MapWrapperVector.java @@ -26,7 +26,6 @@ * Vector wrapping a given map. */ public class MapWrapperVector extends AbstractVector { - /** * Construct a vector wrapping given map. * diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseLocalVector.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseLocalVector.java index be5d0f68bf1fe..9e345bb8ada67 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseLocalVector.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseLocalVector.java @@ -37,10 +37,10 @@ public SparseLocalVector() { /** * @param map Underlying map. - * @param copy Should given map be copied. + * @param cp Should given map be copied. */ - public SparseLocalVector(Map map, boolean copy) { - setStorage(new SparseLocalOnHeapVectorStorage(map, copy)); + public SparseLocalVector(Map map, boolean cp) { + setStorage(new SparseLocalOnHeapVectorStorage(map, cp)); } /** diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/VectorView.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/VectorView.java index 09880597e5201..071c6ed82611f 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/VectorView.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/VectorView.java @@ -60,7 +60,7 @@ private DelegateVectorStorage storage() { /** {@inheritDoc} */ @Override public Vector copy() { - // TODO: revise this + // TODO: IGNTIE-5723, revise this DelegateVectorStorage sto = storage(); return new VectorView(sto.delegate(), sto.offset(), sto.length()); diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java index 7a90ba153e276..752929d4cdd8e 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java @@ -53,7 +53,7 @@ public static Matrix like(Matrix matrix) { */ public static Matrix identityLike(Matrix matrix, int n) { Matrix res = like(matrix, n, n); - // TODO: Maybe we should introduce API for walking(and changing) matrix in + // TODO: IGNITE-5216, Maybe we should introduce API for walking(and changing) matrix in. // a fastest possible visiting order. for (int i = 0; i < n; i++) res.setX(i, i, 1.0); @@ -143,7 +143,7 @@ public static DenseLocalOnHeapMatrix fromList(List vecs, boolean entries return res; } - /** TODO: rewrite in a more optimal way. */ + /** TODO: IGNTIE-5723, rewrite in a more optimal way. */ public static DenseLocalOnHeapVector localCopyOf(Vector vec) { DenseLocalOnHeapVector res = new DenseLocalOnHeapVector(vec.size()); diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/AbstractMultipleLinearRegression.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/AbstractMultipleLinearRegression.java index d558dc08816a9..a2a8f16afaa2d 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/AbstractMultipleLinearRegression.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/AbstractMultipleLinearRegression.java @@ -138,7 +138,7 @@ protected void newYSampleData(Vector y) { throw new NullArgumentException(); if (y.size() == 0) throw new NoDataException(); - // TODO: Should we copy here? + // TODO: IGNITE-5826, Should we copy here? yVector = y; } @@ -174,7 +174,7 @@ protected void newXSampleData(Matrix x) { if (x.rowSize() == 0) throw new NoDataException(); if (noIntercept) - // TODO: Should we copy here? + // TODO: IGNITE-5826, Should we copy here? xMatrix = x; else { // Augment design matrix with initial unitary column xMatrix = MatrixUtil.like(x, x.rowSize(), x.columnSize() + 1); diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/OLSMultipleLinearRegression.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/OLSMultipleLinearRegression.java index c53a77e7d88df..36d5f2c350f5c 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/OLSMultipleLinearRegression.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/OLSMultipleLinearRegression.java @@ -155,7 +155,7 @@ public double calculateTotalSumOfSquares() { if (isNoIntercept()) return getY().foldMap(Functions.PLUS, Functions.SQUARE, 0.0); else { - // TODO: think about incremental update formula. + // TODO: IGNITE-5826, think about incremental update formula. final double mean = getY().sum() / getY().size(); return getY().foldMap(Functions.PLUS, x -> (mean - x) * (mean - x), 0.0); } diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/TestUtils.java b/modules/ml/src/test/java/org/apache/ignite/ml/TestUtils.java index 9c3b4c9555722..62fdf2c2fbf5c 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/TestUtils.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/TestUtils.java @@ -160,7 +160,7 @@ public static void assertEquals(Matrix exp, Matrix actual) { double eij = exp.getX(i, j); double aij = actual.getX(i, j); - // TODO: Check precision here. + // TODO: IGNITE-5824, Check precision here. Assert.assertEquals(eij, aij, 0.0); } } diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/benchmark/MathBenchmark.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/benchmark/MathBenchmark.java index 33ccfc91f8231..69737f71bdf9b 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/benchmark/MathBenchmark.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/benchmark/MathBenchmark.java @@ -197,7 +197,7 @@ private void validate() { /** */ interface BenchmarkCode { - // todo find out why Callable failed to work here + //TODO: IGNITE-5827, find out why Callable failed to work here /** */ void call() throws Exception; diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/benchmark/VectorBenchmarkTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/benchmark/VectorBenchmarkTest.java index fd983826e94cb..98a93d9186de0 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/benchmark/VectorBenchmarkTest.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/benchmark/VectorBenchmarkTest.java @@ -30,7 +30,7 @@ /** */ public class VectorBenchmarkTest { - // todo add benchmarks for other methods in Vector and for other types of Vector and Matrix + //TODO: IGNITE-5827, add benchmarks for other methods in Vector and for other types of Vector and Matrix /** */ @Test diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/decompositions/EigenDecompositionTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/decompositions/EigenDecompositionTest.java index d283ce750b3ef..76aca0ba6d48e 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/decompositions/EigenDecompositionTest.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/decompositions/EigenDecompositionTest.java @@ -78,7 +78,7 @@ public void testNonSquareMatrix() { {0.0d, 1.0d, 0.0d}, {0.0d, 0.0d, 2.0d}, {1.0d, 1.0d, 0.0d}})); - // todo find out why decomposition of 3X4 matrix throws row index exception + // TODO: IGNITE-5828, find out why decomposition of 3X4 matrix throws row index exception Matrix d = decomposition.getD(); Matrix v = decomposition.getV(); diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/MatrixAttributeTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/MatrixAttributeTest.java index 79cb13a9c8795..c645bd75d8fbe 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/MatrixAttributeTest.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/MatrixAttributeTest.java @@ -27,8 +27,6 @@ /** * Attribute tests for matrices. - * - * TODO: WIP */ public class MatrixAttributeTest { /** */ diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/storage/matrix/MatrixStorageImplementationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/storage/matrix/MatrixStorageImplementationTest.java index 72daecaad932c..7f1b6b380f973 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/storage/matrix/MatrixStorageImplementationTest.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/storage/matrix/MatrixStorageImplementationTest.java @@ -28,7 +28,7 @@ /** * Unit tests for {@link MatrixStorage} implementations. * - * TODO: add attribute tests. + * TODO: IGNITE-5723, add attribute tests. */ public class MatrixStorageImplementationTest extends ExternalizeTest { /** diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/AbstractVectorTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/AbstractVectorTest.java index a7954cd0190ee..be7532f1f8e3d 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/AbstractVectorTest.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/AbstractVectorTest.java @@ -357,7 +357,7 @@ public void getElement() { */ @SuppressWarnings("ClassWithoutNoArgConstructor") private AbstractVector getAbstractVector(VectorStorage storage) { - return new AbstractVector(storage) { // TODO: find out how to fix warning about missing constructor + return new AbstractVector(storage) { // TODO: IGNTIE-5723, find out how to fix warning about missing constructor /** {@inheritDoc} */ @Override public boolean isDense() { return false; @@ -426,7 +426,7 @@ private AbstractVector getAbstractVector(VectorStorage storage) { * @return AbstractVector. */ private AbstractVector getAbstractVector() { - return new AbstractVector() { // TODO: find out how to fix warning about missing constructor + return new AbstractVector() { // TODO: IGNTIE-5723, find out how to fix warning about missing constructor /** {@inheritDoc} */ @Override public boolean isDense() { return false; diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/PivotedVectorViewConstructorTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/PivotedVectorViewConstructorTest.java index 72be3244e353a..f07a3fdd91508 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/PivotedVectorViewConstructorTest.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/PivotedVectorViewConstructorTest.java @@ -177,7 +177,7 @@ private static class SampleParams { } /** */ - private static class Metric { // todo consider if softer tolerance (like say 0.1 or 0.01) would make sense here + private static class Metric { // TODO: IGNITE-5824, consider if softer tolerance (like say 0.1 or 0.01) would make sense here /** */ private final double exp; diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/VectorAttributesTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/VectorAttributesTest.java index e2f6a40806fcd..52232ca5c0663 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/VectorAttributesTest.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/VectorAttributesTest.java @@ -75,7 +75,7 @@ public class VectorAttributesTest { "isRandomAccess", "isDistributed"), new Specification(new MatrixVectorView(new DenseLocalOnHeapMatrix(1, 1), 0, 0, 1, 1), DenseLocalOnHeapVector.class, "isDense", - "isRandomAccess", "isDistributed"), // todo find out why "isSequentialAccess" fails here + "isRandomAccess", "isDistributed"), // TODO: IGNTIE-5723, find out why "isSequentialAccess" fails here new Specification(new MatrixVectorView(new DenseLocalOffHeapMatrix(1, 1), 0, 0, 1, 1), DenseLocalOffHeapVector.class, "isDense", "isRandomAccess", "isDistributed")); diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/VectorFoldMapTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/VectorFoldMapTest.java index 676bb17a2eaaa..d18dff20c2e01 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/VectorFoldMapTest.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/VectorFoldMapTest.java @@ -101,7 +101,7 @@ private void toDoubleTest(Function calcRef, Function operation, BiFunction vecOperation) { consumeSampleVectors((v, desc) -> { - // TODO find out if more elaborate testing scenario is needed or it's okay as is. + // TODO: IGNTIE-5723, find out if more elaborate testing scenario is needed or it's okay as is. final int size = v.size(); final double[] ref = new double[size]; diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/VectorImplementationsTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/VectorImplementationsTest.java index 48dcc3608993f..8075b294354d7 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/VectorImplementationsTest.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/VectorImplementationsTest.java @@ -39,7 +39,7 @@ import static org.junit.Assert.fail; /** See also: {@link AbstractVectorTest} and {@link VectorToMatrixTest}. */ -public class VectorImplementationsTest { // todo split this to smaller cohesive test classes +public class VectorImplementationsTest { // TODO: IGNTIE-5723, split this to smaller cohesive test classes /** */ @Test public void vectorImplementationsFixturesTest() { @@ -95,7 +95,7 @@ public void incrementXTest() { public void operateXOutOfBoundsTest() { consumeSampleVectors((v, desc) -> { if (v instanceof DenseLocalOffHeapVector || v instanceof SparseLocalVector || v instanceof SparseLocalOffHeapVector) - return; // todo find out if it's OK to skip by instances here + return; // TODO: IGNTIE-5723, find out if it's OK to skip by instances here boolean expECaught = false; @@ -485,7 +485,7 @@ public void externalizeTest() { @Override public void externalizeTest() { consumeSampleVectors((v, desc) -> { if (v instanceof SparseLocalOffHeapVector) - return; //TODO: wait till SparseLocalOffHeapVector externalization support. + return; //TODO: IGNITE-5801, wait till SparseLocalOffHeapVector externalization support. externalizeTest(v); }); @@ -501,7 +501,7 @@ public void hashCodeTest() { /** */ private boolean getXOutOfBoundsOK(Vector v) { - // todo find out if this is indeed OK + // TODO: IGNTIE-5723, find out if this is indeed OK return v instanceof RandomVector || v instanceof ConstantVector || v instanceof SingleElementVector || v instanceof SingleElementVectorView; } @@ -569,7 +569,7 @@ private void toDoubleTest(Function calcRef, Function operation, BiFunction vecOperation) { consumeSampleVectors((v, desc) -> { - // TODO find out if more elaborate testing scenario is needed or it's okay as is. + // TODO : IGNTIE-5723, find out if more elaborate testing scenario is needed or it's okay as is. final int size = v.size(); final double[] ref = new double[size]; @@ -827,7 +827,7 @@ private int generated(int idx) { } /** */ - static class Metric { // todo consider if softer tolerance (like say 0.1 or 0.01) would make sense here + static class Metric { //TODO: IGNITE-5824, consider if softer tolerance (like say 0.1 or 0.01) would make sense here /** */ private final double exp; diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/VectorIterableTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/VectorIterableTest.java index 16c2571865cca..6400c80a7cdf9 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/VectorIterableTest.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/VectorIterableTest.java @@ -148,7 +148,7 @@ && isZero(nonZeroesOddVec.getElement(0).get()) /** */ @Test public void nonZeroesTest() { - // todo make RandomVector constructor that accepts a function and use it here + // TODO: IGNTIE-5723, make RandomVector constructor that accepts a function and use it here. // in order to *reliably* test non-zeroes in there consumeSampleVectors( (v, desc) -> consumeSampleVectorsWithZeroes(v, (vec, numZeroes) diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/VectorToMatrixTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/VectorToMatrixTest.java index 4d5bc56c0eece..a003dcfd62a85 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/VectorToMatrixTest.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/vector/VectorToMatrixTest.java @@ -268,7 +268,7 @@ private static Map, Class> typesMap() put(DenseLocalOffHeapVector.class, DenseLocalOffHeapMatrix.class); put(RandomVector.class, RandomMatrix.class); put(SparseLocalVector.class, SparseLocalOnHeapMatrix.class); - put(SingleElementVector.class, null); // todo find out if we need SingleElementMatrix to match, or skip it + put(SingleElementVector.class, null); // TODO: IGNTIE-5723, find out if we need SingleElementMatrix to match, or skip it. put(ConstantVector.class, null); put(FunctionVector.class, null); put(PivotedVectorView.class, DenseLocalOnHeapMatrix.class); // IMPL NOTE per fixture @@ -280,7 +280,7 @@ private static Map, Class> typesMap() } /** */ - private static class Metric { // todo consider if softer tolerance (like say 0.1 or 0.01) would make sense here + private static class Metric { //TODO: IGNITE-5824, consider if softer tolerance (like say 0.1 or 0.01) would make sense here. /** */ private final double exp; diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/OLSMultipleLinearRegressionTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/OLSMultipleLinearRegressionTest.java index 7f76750a61647..c0069b5286142 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/OLSMultipleLinearRegressionTest.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/OLSMultipleLinearRegressionTest.java @@ -186,7 +186,7 @@ public void testLongly() { Assert.assertEquals(0.995479004577296, mdl.calculateRSquared(), 1E-12); Assert.assertEquals(0.992465007628826, mdl.calculateAdjustedRSquared(), 1E-12); - // TODO: uncomment + // TODO: IGNITE-5826, uncomment. // checkVarianceConsistency(model); // Estimate model without intercept @@ -336,7 +336,7 @@ public void testSwissFertility() { Assert.assertEquals(0.649789742860228, mdl.calculateRSquared(), 1E-12); Assert.assertEquals(0.6164363850373927, mdl.calculateAdjustedRSquared(), 1E-12); - // TODO: uncomment + // TODO: IGNITE-5826, uncomment. // checkVarianceConsistency(model); // Estimate the model with no intercept From 00babb4bcb0d432ae614a3cb98dfa7a61dfdbc1c Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Tue, 25 Jul 2017 15:50:43 +0300 Subject: [PATCH 024/547] IGNITE-5829 Linked TODO with ticket. --- .../processors/cache/persistence/freelist/FreeListImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java index e99a5eecc1b70..d50020e553f98 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java @@ -193,7 +193,7 @@ private int addRow( io.addRow(pageAddr, row, rowSize, pageSize()); if (needWalDeltaRecord(pageId, page, null)) { - // TODO This record must contain only a reference to a logical WAL record with the actual data. + // TODO IGNITE-5829 This record must contain only a reference to a logical WAL record with the actual data. byte[] payload = new byte[rowSize]; DataPagePayload data = io.readPayload(pageAddr, PageIdUtils.itemId(row.link()), pageSize()); @@ -239,7 +239,7 @@ private int addRowFragment( assert payloadSize > 0 : payloadSize; if (needWalDeltaRecord(pageId, page, null)) { - // TODO This record must contain only a reference to a logical WAL record with the actual data. + // TODO IGNITE-5829 This record must contain only a reference to a logical WAL record with the actual data. byte[] payload = new byte[payloadSize]; DataPagePayload data = io.readPayload(pageAddr, PageIdUtils.itemId(row.link()), pageSize()); From 243913ee24fbc09d0ae5469bb1d99926b1a8e8d5 Mon Sep 17 00:00:00 2001 From: EdShangGG Date: Tue, 25 Jul 2017 16:50:35 +0300 Subject: [PATCH 025/547] IGNITE-5747 GridCommonAbstractTest.startGridsMultiThreaded works very slowly if persistence is enabled - Fixes #2294. Signed-off-by: Alexey Goncharuk --- .../testframework/junits/common/GridCommonAbstractTest.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java index dc7e89d6815cb..014103dada7ea 100755 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java @@ -503,8 +503,12 @@ protected GridNearCacheAdapter near(int idx, String cache) { protected final Ignite startGridsMultiThreaded(int cnt, boolean awaitPartMapExchange) throws Exception { Ignite g = super.startGridsMultiThreaded(cnt); - if (awaitPartMapExchange) + if (awaitPartMapExchange) { + if (!g.active()) + g.active(true); + awaitPartitionMapExchange(); + } return g; } From de259fffb7fa5c0f8f6f7eeb84b86f296fe3bde9 Mon Sep 17 00:00:00 2001 From: Yury Babak Date: Tue, 25 Jul 2017 20:19:27 +0300 Subject: [PATCH 026/547] IGNITE-5278: BLAS implemented. --- LICENSE | 7 + .../ml/math/matrix/ExampleMatrixStorage.java | 6 +- modules/ml/README.txt | 6 + modules/ml/licenses/bsd3.txt | 51 ++ modules/ml/pom.xml | 22 + .../java/org/apache/ignite/ml/math/Blas.java | 484 ++++++++++++++++++ .../org/apache/ignite/ml/math/Matrix.java | 31 ++ .../apache/ignite/ml/math/MatrixStorage.java | 2 +- .../apache/ignite/ml/math/OrderedMatrix.java | 24 + .../org/apache/ignite/ml/math/Vector.java | 8 + .../decompositions/CholeskyDecomposition.java | 11 +- .../IgniteIntDoubleToDoubleBiFunction.java | 27 + .../IgniteIntIntToIntBiFunction.java | 27 + .../ml/math/functions/IgniteTriFunction.java | 35 ++ .../ml/math/impls/matrix/AbstractMatrix.java | 94 +++- .../impls/matrix/DenseLocalOnHeapMatrix.java | 61 ++- .../impls/matrix/SparseLocalOnHeapMatrix.java | 27 + .../storage/matrix/ArrayMatrixStorage.java | 78 ++- .../matrix/DenseOffHeapMatrixStorage.java | 2 +- .../storage/matrix/MatrixDelegateStorage.java | 2 +- .../SparseDistributedMatrixStorage.java | 4 +- .../SparseLocalOnHeapMatrixStorage.java | 18 + .../SparseLocalOnHeapVectorStorage.java | 6 + .../impls/vector/AbstractReadOnlyVector.java | 6 + .../ml/math/impls/vector/AbstractVector.java | 8 + .../math/impls/vector/DelegatingVector.java | 6 + .../math/impls/vector/SparseLocalVector.java | 39 ++ .../ignite/ml/math/util/MatrixUtil.java | 48 ++ .../KMeansDistributedClustererTest.java | 15 +- .../org/apache/ignite/ml/math/BlasTest.java | 357 +++++++++++++ .../ignite/ml/math/MathImplMainTestSuite.java | 3 +- .../matrix/MatrixViewConstructorTest.java | 2 +- .../matrix/MatrixArrayStorageTest.java | 6 +- 33 files changed, 1475 insertions(+), 48 deletions(-) create mode 100644 modules/ml/licenses/bsd3.txt create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/math/Blas.java create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/math/OrderedMatrix.java create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteIntDoubleToDoubleBiFunction.java create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteIntIntToIntBiFunction.java create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteTriFunction.java create mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/math/BlasTest.java diff --git a/LICENSE b/LICENSE index c971434936359..6e77825327930 100644 --- a/LICENSE +++ b/LICENSE @@ -219,6 +219,13 @@ This product bundles SnapTree, which is available under a "3-clause BSD" license. For details, see https://github.com/nbronson/snaptree/blob/master/LICENSE. +============================================================================== +For netlib-java: +============================================================================== +This product bundles netlib-java, which is available under a +"3-clause BSD" license. For details, see +https://github.com/fommil/netlib-java/blob/master/LICENSE.txt. + ============================================================================== For JSR 166 classes in "org.jsr166" package ============================================================================== diff --git a/examples/src/main/ml/org/apache/ignite/examples/ml/math/matrix/ExampleMatrixStorage.java b/examples/src/main/ml/org/apache/ignite/examples/ml/math/matrix/ExampleMatrixStorage.java index 5dedfbfc98b4e..1ccb2e2c9e4e1 100644 --- a/examples/src/main/ml/org/apache/ignite/examples/ml/math/matrix/ExampleMatrixStorage.java +++ b/examples/src/main/ml/org/apache/ignite/examples/ml/math/matrix/ExampleMatrixStorage.java @@ -23,7 +23,9 @@ import java.util.Arrays; import org.apache.ignite.ml.math.MatrixStorage; +import org.apache.ignite.ml.math.StorageConstants; import org.apache.ignite.ml.math.impls.storage.matrix.ArrayMatrixStorage; +import org.apache.ignite.ml.math.util.MatrixUtil; /** * Example matrix storage implementation, modeled after {@link ArrayMatrixStorage}. @@ -117,8 +119,8 @@ public ExampleMatrixStorage() { } /** {@inheritDoc} */ - @Override public double[][] data() { - return data; + @Override public double[] data() { + return MatrixUtil.flatten(data, StorageConstants.ROW_STORAGE_MODE); } /** {@inheritDoc */ diff --git a/modules/ml/README.txt b/modules/ml/README.txt index e0cc093457e31..e85b7a0118dc6 100644 --- a/modules/ml/README.txt +++ b/modules/ml/README.txt @@ -12,4 +12,10 @@ Based on ideas from Apache Mahout. Run from project root: mvn clean package -Pml -DskipTests -pl modules/ml -am +Apache binary releases cannot include LGPL dependencies. If you would like to activate native BLAS optimizations +into your build, you should download the source release +from Ignite website and do the build with the following maven command: + +mvn clean package -Pml,lgpl -DskipTests -pl modules/ml -am + Find generated jars in target folder. \ No newline at end of file diff --git a/modules/ml/licenses/bsd3.txt b/modules/ml/licenses/bsd3.txt new file mode 100644 index 0000000000000..d6b30c1c933b8 --- /dev/null +++ b/modules/ml/licenses/bsd3.txt @@ -0,0 +1,51 @@ +This product binaries redistribute netlib-java which is available under the following license: + +Copyright (c) 2013 Samuel Halliday +Copyright (c) 1992-2011 The University of Tennessee and The University + of Tennessee Research Foundation. All rights + reserved. +Copyright (c) 2000-2011 The University of California Berkeley. All + rights reserved. +Copyright (c) 2006-2011 The University of Colorado Denver. All rights + reserved. + +$COPYRIGHT$ + +Additional copyrights may follow + +$HEADER$ + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +- Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +- Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer listed + in this license in the documentation and/or other materials + provided with the distribution. + +- Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +The copyright holders provide no reassurances that the source code +provided does not infringe any patent, copyright, or any other +intellectual property rights of third parties. The copyright holders +disclaim any liability to any recipient for claims brought against +recipient by any third party for infringement of that parties +intellectual property rights. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/modules/ml/pom.xml b/modules/ml/pom.xml index d6197199fd6be..478d8c422e2fe 100644 --- a/modules/ml/pom.xml +++ b/modules/ml/pom.xml @@ -36,6 +36,7 @@ UTF-8 + 1.1.2 @@ -87,6 +88,13 @@ test + + com.github.fommil.netlib + core + ${netlibjava.version} + pom + + org.apache.commons commons-rng-core @@ -101,6 +109,20 @@ + + + lgpl + + + + com.github.fommil.netlib + all + ${netlibjava.version} + + + + + diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/Blas.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/Blas.java new file mode 100644 index 0000000000000..57f994e460321 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/Blas.java @@ -0,0 +1,484 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math; + +import com.github.fommil.netlib.BLAS; +import com.github.fommil.netlib.F2jBLAS; +import it.unimi.dsi.fastutil.ints.Int2ObjectArrayMap; +import it.unimi.dsi.fastutil.ints.IntIterator; +import it.unimi.dsi.fastutil.ints.IntSet; +import java.util.Set; +import org.apache.ignite.ml.math.exceptions.CardinalityException; +import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException; +import org.apache.ignite.ml.math.exceptions.NonSquareMatrixException; +import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix; +import org.apache.ignite.ml.math.impls.matrix.SparseLocalOnHeapMatrix; +import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector; +import org.apache.ignite.ml.math.impls.vector.SparseLocalVector; + +/** + * Useful subset of BLAS operations. + * This class is based on 'BLAS' class from Apache Spark MLlib. + */ +public class Blas { + /** F2J implementation of BLAS. */ + transient static private BLAS f2jBlas = new F2jBLAS(); + + /** Native implementation of BLAS. F2J implementation will be used as fallback if no native implementation is found + */ + transient static private BLAS nativeBlas = BLAS.getInstance(); + + /** + * Performs y += a * x + * + * @param a Scalar a. + * @param x Vector x. + * @param y Vector y. + */ + public static void axpy(Double a, Vector x, Vector y) { + if (x.size() != y.size()) + throw new CardinalityException(x.size(), y.size()); + + if (x.isArrayBased() && y.isArrayBased()) + axpy(a, x.getStorage().data(), y.getStorage().data()); + else if (x instanceof SparseLocalVector && y.isArrayBased()) + axpy(a, (SparseLocalVector)x, y.getStorage().data()); + else + throw new MathIllegalArgumentException("Operation 'axpy' doesn't support this combination of parameters [x=" + + x.getClass().getName() + ", y="+y.getClass().getName()+"]."); + } + + /** */ + private static void axpy(Double a, double[] x, double[] y) { + f2jBlas.daxpy(x.length, a, x, 1, y, 1); + } + + /** */ + private static void axpy(Double a, SparseLocalVector x, double[] y) { + int xSize = x.size(); + + if (a == 1.0) { + int k = 0; + + while (k < xSize) { + y[k] += x.getX(k); + k++; + } + } else { + int k = 0; + + while (k < xSize) { + y[k] += a * x.getX(k); + k++; + } + } + } + + /** + * Returns dot product of vectors x and y. + * + * @param x Vector x. + * @param y Vector y. + * @return Dot product of x and y. + **/ + public static Double dot(Vector x, Vector y) { + return x.dot(y); + } + + /** + * Copies Vector x into Vector y. (y = x) + * + * @param x Vector x. + * @param y Vector y. + */ + public void copy(Vector x, Vector y) { + int n = y.size(); + + if (x.size() != n) + throw new CardinalityException(x.size(), n); + + if (y.isArrayBased()) { + double[] yData = y.getStorage().data(); + + if (x.isArrayBased()) + System.arraycopy(x.getStorage().data(), 0, y.getStorage().data(), 0, n); + else { + if (y instanceof SparseLocalVector) { + for (int i = 0; i < n; i++) + yData[i] = x.getX(i); + } + } + } else + throw new IllegalArgumentException("Vector y must be array based in copy."); + } + + + /** + * Performs in-place multiplication of vector x by a real scalar a. (x = a * x) + * + * @param a Scalar a. + * @param x Vector x. + **/ + public static void scal(Double a, Vector x) { + if (x.isArrayBased()) + f2jBlas.dscal(x.size(), a, x.getStorage().data(), 1); + else if (x instanceof SparseLocalVector) { + Set indexes = ((SparseLocalVector)x).indexes(); + + for (Integer i : indexes) + x.compute(i, (ind, v) -> v * a); + } else + throw new IllegalArgumentException(); + } + + /** + * Adds alpha * v * v.t to a matrix in-place. This is the same as BLAS's ?SPR. + * + * @param u the upper triangular part of the matrix in a [[DenseVector]](column major) + */ + public static void spr(Double alpha, DenseLocalOnHeapVector v, DenseLocalOnHeapVector u) { + nativeBlas.dspr("U", v.size(), alpha, v.getStorage().data(), 1, u.getStorage().data()); + } + + /** */ + public static void spr(Double alpha, SparseLocalVector v, DenseLocalOnHeapVector u) { + int prevNonDfltInd = 0; + int startInd = 0; + double av; + double[] uData = u.getStorage().data(); + + for (Integer nonDefaultInd : v.indexes()) { + startInd += (nonDefaultInd - prevNonDfltInd) * (nonDefaultInd + prevNonDfltInd + 1) / 2; + av = alpha * v.get(nonDefaultInd); + + for (Integer i : v.indexes()) + if (i <= nonDefaultInd) + uData[startInd + i] += av * v.getX(i); + + prevNonDfltInd = nonDefaultInd; + } + } + + /** + * A := alpha * x * x^T + A + * @param alpha a real scalar that will be multiplied to x * x^T^. + * @param x the vector x that contains the n elements. + * @param a the symmetric matrix A. Size of n x n. + */ + void syr(Double alpha, Vector x, DenseLocalOnHeapMatrix a) { + int mA = a.rowSize(); + int nA = a.columnSize(); + + if (mA != nA) + throw new NonSquareMatrixException(mA, nA); + + if (mA != x.size()) + throw new CardinalityException(x.size(), mA); + + // TODO: IGNITE-5535, Process DenseLocalOffHeapVector + if (x instanceof DenseLocalOnHeapVector) + syr(alpha, x, a); + else if (x instanceof SparseLocalVector) + syr(alpha, x, a); + else + throw new IllegalArgumentException("Operation 'syr' does not support vector [class=" + + x.getClass().getName() + "]."); + } + + /** */ + static void syr(Double alpha, DenseLocalOnHeapVector x, DenseLocalOnHeapMatrix a) { + int nA = a.rowSize(); + int mA = a.columnSize(); + + nativeBlas.dsyr("U", x.size(), alpha, x.getStorage().data(), 1, a.getStorage().data(), nA); + + // Fill lower triangular part of A + int i = 0; + while (i < mA) { + int j = i + 1; + + while (j < nA) { + a.setX(i, j, a.getX(j, i)); + j++; + } + i++; + } + } + + /** */ + public static void syr(Double alpha, SparseLocalVector x, DenseLocalOnHeapMatrix a) { + int mA = a.columnSize(); + + for (Integer i : x.indexes()) { + double mult = alpha * x.getX(i); + for (Integer j : x.indexes()) + a.getStorage().data()[mA * i + j] += mult * x.getX(j); + } + } + + /** + * For the moment we have no flags indicating if matrix is transposed or not. Therefore all dgemm parameters for + * transposition are equal to 'N'. + */ + public static void gemm(Double alpha, Matrix a, DenseLocalOnHeapMatrix b, Double beta, DenseLocalOnHeapMatrix c) { + if (alpha == 0.0 && beta == 1.0) + return; + else if (alpha == 0.0) + scal(c, beta); + else { + if (a instanceof SparseLocalOnHeapMatrix) + gemm(alpha, (SparseLocalOnHeapMatrix)a, b, beta, c); + else if (a instanceof DenseLocalOnHeapMatrix) { + double[] fA = a.getStorage().data(); + double[] fB = b.getStorage().data(); + double[] fC = c.getStorage().data(); + + nativeBlas.dgemm("N", "N", a.rowSize(), b.columnSize(), a.columnSize(), alpha, fA, + a.rowSize(), fB, b.rowSize(), beta, fC, c.rowSize()); + } else + throw new IllegalArgumentException("Operation 'gemm' doesn't support for matrix [class=" + + a.getClass().getName() + "]."); + } + } + + /** + * C := alpha * A * B + beta * C + * For `SparseMatrix` A. + */ + private static void gemm(Double alpha, SparseLocalOnHeapMatrix a, DenseLocalOnHeapMatrix b, Double beta, + DenseLocalOnHeapMatrix c) { + int mA = a.rowSize(); + int nB = b.columnSize(); + int kA = a.columnSize(); + int kB = b.rowSize(); + + if (kA != kB) + throw new CardinalityException(kA, kB); + + if (mA != c.rowSize()) + throw new CardinalityException(mA, c.rowSize()); + + if (nB != c.columnSize()) + throw new CardinalityException(nB, c.columnSize()); + + if (beta != 1.0) + scal(c, beta); + + Int2ObjectArrayMap im = a.indexesMap(); + IntIterator rowsIter = im.keySet().iterator(); + int row; + // We use here this form of iteration instead of 'for' because of nextInt. + while (rowsIter.hasNext()) { + row = rowsIter.nextInt(); + + for (int colInd = 0; colInd < nB; colInd++) { + double sum = 0.0; + + IntIterator kIter = im.get(row).iterator(); + int k; + + while (kIter.hasNext()) { + k = kIter.nextInt(); + sum += a.get(row, k) * b.get(k, colInd) * alpha; + } + + c.setX(row, colInd, c.getX(row, colInd) + sum); + } + } + } + + /** + * y := alpha * A * x + beta * y. + * + * @param alpha Alpha. + * @param a Matrix a. + * @param x Vector x. + * @param beta Beta. + * @param y Vector y. + */ + public static void gemv(double alpha, Matrix a, Vector x, double beta, DenseLocalOnHeapVector y) { + checkCardinality(a, x); + checkCardinality(a, y); + + if (alpha == 0.0 && beta == 1.0) + return; + + if (alpha == 0.0) { + scal(y, beta); + return; + } + + if (a instanceof SparseLocalOnHeapMatrix && x instanceof DenseLocalOnHeapVector) + gemv(alpha, (SparseLocalOnHeapMatrix)a, (DenseLocalOnHeapVector)x, beta, y); + else if (a instanceof SparseLocalOnHeapMatrix && x instanceof SparseLocalVector) + gemv(alpha, (SparseLocalOnHeapMatrix)a, (SparseLocalVector)x, beta, y); + else if (a instanceof DenseLocalOnHeapMatrix && x instanceof DenseLocalOnHeapVector) + gemv(alpha, (DenseLocalOnHeapMatrix)a, (DenseLocalOnHeapVector)x, beta, y); + else if (a instanceof DenseLocalOnHeapMatrix && x instanceof SparseLocalVector) + gemv(alpha, (DenseLocalOnHeapMatrix)a, (SparseLocalVector)x, beta, y); + else + throw new IllegalArgumentException("Operation gemv doesn't support running thist input [matrix=" + + a.getClass().getSimpleName() + ", vector=" + x.getClass().getSimpleName()+"]."); + } + + /** + * y := alpha * A * x + beta * y. + * + * @param alpha Alpha. + * @param a Matrix a. + * @param x Vector x. + * @param beta Beta. + * @param y Vector y. + */ + private static void gemv(double alpha, SparseLocalOnHeapMatrix a, DenseLocalOnHeapVector x, double beta, + DenseLocalOnHeapVector y) { + + if (beta != 1.0) + scal(y, beta); + + IntIterator rowIter = a.indexesMap().keySet().iterator(); + while (rowIter.hasNext()) { + int row = rowIter.nextInt(); + + double sum = 0.0; + IntIterator colIter = a.indexesMap().get(row).iterator(); + while (colIter.hasNext()) { + int col = colIter.nextInt(); + sum += alpha * a.getX(row, col) * x.getX(col); + } + + y.setX(row, y.getX(row) + sum); + } + } + + /** + * y := alpha * A * x + beta * y. + * + * @param alpha Alpha. + * @param a Matrix a. + * @param x Vector x. + * @param beta Beta. + * @param y Vector y. + */ + private static void gemv(double alpha, DenseLocalOnHeapMatrix a, DenseLocalOnHeapVector x, double beta, + DenseLocalOnHeapVector y) { + nativeBlas.dgemv("N", a.rowSize(), a.columnSize(), alpha, a.getStorage().data(), a.rowSize(), x.getStorage().data(), 1, beta, + y.getStorage().data(), 1); + } + + /** + * y := alpha * A * x + beta * y. + * + * @param alpha Alpha. + * @param a Matrix a. + * @param x Vector x. + * @param beta Beta. + * @param y Vector y. + */ + private static void gemv(double alpha, SparseLocalOnHeapMatrix a, SparseLocalVector x, double beta, + DenseLocalOnHeapVector y) { + + + if (beta != 1.0) + scal(y, beta); + + IntIterator rowIter = a.indexesMap().keySet().iterator(); + while (rowIter.hasNext()) { + int row = rowIter.nextInt(); + + double sum = 0.0; + IntIterator colIter = a.indexesMap().get(row).iterator(); + while (colIter.hasNext()) { + int col = colIter.nextInt(); + + sum += alpha * a.getX(row, col) * x.getX(col); + } + + y.set(row, y.get(row) + sum); + } + } + + /** + * y := alpha * A * x + beta * y. + * + * @param alpha Alpha. + * @param a Matrix a. + * @param x Vector x. + * @param beta Beta. + * @param y Vector y. + */ + private static void gemv(double alpha, DenseLocalOnHeapMatrix a, SparseLocalVector x, double beta, + DenseLocalOnHeapVector y) { + int rowCntrForA = 0; + int mA = a.rowSize(); + + double[] aData = a.getStorage().data(); + + IntSet indexes = x.indexes(); + + double[] yValues = y.getStorage().data(); + + while (rowCntrForA < mA) { + double sum = 0.0; + + IntIterator iter = indexes.iterator(); + while (iter.hasNext()) { + int xIdx = iter.nextInt(); + sum += x.getX(xIdx) * aData[xIdx * mA + rowCntrForA]; + } + + yValues[rowCntrForA] = sum * alpha + beta * yValues[rowCntrForA]; + rowCntrForA++; + } + } + + /** + * M := alpha * M. + * @param m Matrix M. + * @param alpha Aplha. + */ + private static void scal(Matrix m, double alpha) { + if (alpha != 1.0) + for (int i = 0; i < m.rowSize(); i++) + for (int j = 0; j < m.columnSize(); j++) + m.setX(i, j, m.getX(i, j) * alpha); + + } + + /** + * v := alpha * v. + * @param v Vector v. + * @param alpha Aplha. + */ + private static void scal(Vector v, double alpha) { + if (alpha != 1.0) + for (int i = 0; i < v.size(); i++) + v.compute(i, (ind, val) -> val * alpha); + } + + /** + * Checks if Matrix A can be multiplied by vector v, if not CardinalityException is thrown. + * + * @param a Matrix A. + * @param v Vector v. + */ + public static void checkCardinality(Matrix a, Vector v) throws CardinalityException { + if (a.columnSize() != v.size()) + throw new CardinalityException(a.columnSize(), v.size()); + } +} diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/Matrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/Matrix.java index 2cf4e6381268e..66de1a161a878 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/Matrix.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/Matrix.java @@ -18,6 +18,7 @@ package org.apache.ignite.ml.math; import java.io.Externalizable; +import java.util.Spliterator; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.ml.math.exceptions.CardinalityException; import org.apache.ignite.ml.math.exceptions.IndexException; @@ -25,6 +26,7 @@ import org.apache.ignite.ml.math.functions.IgniteBiFunction; import org.apache.ignite.ml.math.functions.IgniteDoubleFunction; import org.apache.ignite.ml.math.functions.IgniteFunction; +import org.apache.ignite.ml.math.functions.IgniteTriFunction; import org.apache.ignite.ml.math.functions.IntIntToDoubleFunction; /** @@ -186,6 +188,27 @@ interface Element { */ public Matrix map(Matrix mtx, IgniteBiFunction fun); + /** + * Gets number of non-zero elements in this matrix. + * + * @return Number of non-zero elements in this matrix. + */ + public int nonZeroElements(); + + /** + * Gets spliterator for all values in this matrix. + * + * @return Spliterator for all values. + */ + public Spliterator allSpliterator(); + + /** + * Gets spliterator for all non-zero values in this matrix. + * + * @return Spliterator for all non-zero values. + */ + public Spliterator nonZeroSpliterator(); + /** * Assigns values from given vector to the specified column in this matrix. * @@ -515,4 +538,12 @@ interface Element { public default void destroy() { // No-op. } + + /** + * Replace matrix entry with value oldVal at (row, col) with result of computing f(row, col, oldVal). + * @param row Row. + * @param col Column. + * @param f Function used for replacing. + */ + public void compute(int row, int col, IgniteTriFunction f); } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/MatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/MatrixStorage.java index ccfebe5080393..3b905bc08b0ba 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/MatrixStorage.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/MatrixStorage.java @@ -52,7 +52,7 @@ public interface MatrixStorage extends Externalizable, StorageOpsMetrics, Destro * * @see StorageOpsMetrics#isArrayBased() */ - default public double[][] data() { + default public double[] data() { return null; } } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/OrderedMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/OrderedMatrix.java new file mode 100644 index 0000000000000..2c3acc8ac6729 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/OrderedMatrix.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math; + +/** Interface for matrix with particular order for storing entities. */ +public interface OrderedMatrix { + /** Get access mode. */ + public int accessMode(); +} diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/Vector.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/Vector.java index e1c5df06a9ee7..5fd39af97366b 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/Vector.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/Vector.java @@ -26,6 +26,7 @@ import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException; import org.apache.ignite.ml.math.functions.IgniteBiFunction; import org.apache.ignite.ml.math.functions.IgniteDoubleFunction; +import org.apache.ignite.ml.math.functions.IgniteIntDoubleToDoubleBiFunction; /** * A vector interface. @@ -496,4 +497,11 @@ public T foldMap(Vector vec, IgniteBiFunction foldFun, * @return Vector GUID. */ public IgniteUuid guid(); + + /** + * Replace vector entry with value oldVal at i with result of computing f(i, oldVal). + * @param i Position. + * @param f Function used for replacing. + **/ + public void compute(int i, IgniteIntDoubleToDoubleBiFunction f); } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/CholeskyDecomposition.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/CholeskyDecomposition.java index 84028fe32f893..73fbe2cea30a1 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/CholeskyDecomposition.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/CholeskyDecomposition.java @@ -23,6 +23,7 @@ import org.apache.ignite.ml.math.exceptions.CardinalityException; import org.apache.ignite.ml.math.exceptions.NonPositiveDefiniteMatrixException; import org.apache.ignite.ml.math.exceptions.NonSymmetricMatrixException; +import org.apache.ignite.ml.math.util.MatrixUtil; import static org.apache.ignite.ml.math.util.MatrixUtil.like; import static org.apache.ignite.ml.math.util.MatrixUtil.likeVector; @@ -252,7 +253,7 @@ public Matrix solve(final Matrix b) { throw new CardinalityException(b.rowSize(), m); final int nColB = b.columnSize(); - final double[][] x = b.getStorage().data(); + final double[][] x = MatrixUtil.unflatten(b.getStorage().data(), b.columnSize()); // Solve LY = b for (int j = 0; j < m; j++) { @@ -295,15 +296,13 @@ public Matrix solve(final Matrix b) { /** */ private double[][] toDoubleArr(Matrix mtx) { if (mtx.isArrayBased()) - return mtx.getStorage().data(); + return MatrixUtil.unflatten(mtx.getStorage().data(), mtx.columnSize()); - double[][] res = new double[mtx.rowSize()][]; + double[][] res = new double[mtx.rowSize()][mtx.columnSize()]; - for (int row = 0; row < mtx.rowSize(); row++) { - res[row] = new double[mtx.columnSize()]; + for (int row = 0; row < mtx.rowSize(); row++) for (int col = 0; col < mtx.columnSize(); col++) res[row][col] = mtx.get(row, col); - } return res; } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteIntDoubleToDoubleBiFunction.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteIntDoubleToDoubleBiFunction.java new file mode 100644 index 0000000000000..c9a91aea04042 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteIntDoubleToDoubleBiFunction.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.functions; + +import java.io.Serializable; + +/** BiFunction (int, double) -> double. */ +@FunctionalInterface +public interface IgniteIntDoubleToDoubleBiFunction extends Serializable { + /** */ + public double apply(int x, double d); +} diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteIntIntToIntBiFunction.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteIntIntToIntBiFunction.java new file mode 100644 index 0000000000000..bfd472c105374 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteIntIntToIntBiFunction.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.functions; + +import java.io.Serializable; + +/** BiFunction (int, int) -> int. */ +@FunctionalInterface +public interface IgniteIntIntToIntBiFunction extends Serializable { + /** */ + public int apply(int x, int y); +} diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteTriFunction.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteTriFunction.java new file mode 100644 index 0000000000000..3284a00dc4615 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteTriFunction.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.functions; + +import java.io.Serializable; +import java.util.Objects; +import java.util.function.Function; + +/** Serializable TriFunction (A, B, C) -> R. */ +@FunctionalInterface +public interface IgniteTriFunction extends Serializable { + /** */ + R apply(A a, B b, C c); + + /** */ + default IgniteTriFunction andThen(Function after) { + Objects.requireNonNull(after); + return (A a, B b, C c) -> after.apply(apply(a, b, c)); + } +} \ No newline at end of file diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java index 647ebc6dd811b..e1efd0c9b0803 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java @@ -24,6 +24,8 @@ import java.util.HashMap; import java.util.Map; import java.util.Random; +import java.util.Spliterator; +import java.util.function.Consumer; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.ml.math.Matrix; import org.apache.ignite.ml.math.MatrixStorage; @@ -36,6 +38,7 @@ import org.apache.ignite.ml.math.functions.IgniteBiFunction; import org.apache.ignite.ml.math.functions.IgniteDoubleFunction; import org.apache.ignite.ml.math.functions.IgniteFunction; +import org.apache.ignite.ml.math.functions.IgniteTriFunction; import org.apache.ignite.ml.math.functions.IntIntToDoubleFunction; import org.apache.ignite.ml.math.impls.vector.MatrixVectorView; @@ -326,8 +329,7 @@ protected void checkIndex(int row, int col) { /** {@inheritDoc} */ @Override public Matrix assign(double val) { if (sto.isArrayBased()) - for (double[] column : sto.data()) - Arrays.fill(column, val); + Arrays.fill(sto.data(), val); else { int rows = rowSize(); int cols = columnSize(); @@ -420,6 +422,85 @@ private void checkCardinality(int rows, int cols) { return this; } + /** {@inheritDoc} */ + @Override public Spliterator allSpliterator() { + return new Spliterator() { + /** {@inheritDoc} */ + @Override public boolean tryAdvance(Consumer act) { + int rLen = rowSize(); + int cLen = columnSize(); + + for (int i = 0; i < rLen; i++) + for (int j = 0; j < cLen; j++) + act.accept(storageGet(i, j)); + + return true; + } + + /** {@inheritDoc} */ + @Override public Spliterator trySplit() { + return null; // No Splitting. + } + + /** {@inheritDoc} */ + @Override public long estimateSize() { + return rowSize() * columnSize(); + } + + /** {@inheritDoc} */ + @Override public int characteristics() { + return ORDERED | SIZED; + } + }; + } + + /** {@inheritDoc} */ + @Override public int nonZeroElements() { + int cnt = 0; + + for (int i = 0; i < rowSize(); i++) + for (int j = 0; j < rowSize(); j++) + if (get(i, j) != 0.0) + cnt++; + + return cnt; + } + + /** {@inheritDoc} */ + @Override public Spliterator nonZeroSpliterator() { + return new Spliterator() { + /** {@inheritDoc} */ + @Override public boolean tryAdvance(Consumer act) { + int rLen = rowSize(); + int cLen = columnSize(); + + for (int i = 0; i < rLen; i++) + for (int j = 0; j < cLen; j++) { + double val = storageGet(i, j); + + if (val != 0.0) + act.accept(val); + } + return true; + } + + /** {@inheritDoc} */ + @Override public Spliterator trySplit() { + return null; // No Splitting. + } + + /** {@inheritDoc} */ + @Override public long estimateSize() { + return nonZeroElements(); + } + + /** {@inheritDoc} */ + @Override public int characteristics() { + return ORDERED | SIZED; + } + }; + } + /** {@inheritDoc} */ @Override public Matrix assignColumn(int col, Vector vec) { checkColumnIndex(col); @@ -442,7 +523,7 @@ private void checkCardinality(int rows, int cols) { throw new CardinalityException(cols, vec.size()); if (sto.isArrayBased() && vec.getStorage().isArrayBased()) - System.arraycopy(vec.getStorage().data(), 0, sto.data()[row], 0, cols); + System.arraycopy(vec.getStorage().data(), 0, sto.data(), cols * row, cols); else for (int y = 0; y < cols; y++) storageSet(row, y, vec.getX(y)); @@ -623,7 +704,7 @@ protected Matrix likeIdentity() { throw new CardinalityException(cols, data.length); if (sto.isArrayBased()) - System.arraycopy(data, 0, sto.data()[row], 0, cols); + System.arraycopy(data, 0, sto.data(), row * cols, cols); else for (int y = 0; y < cols; y++) setX(row, y, data[y]); @@ -880,4 +961,9 @@ else if (mean > threshold + iv) return (sto != null ? sto.equals(that.getStorage()) : that.getStorage() == null); } + + /** {@inheritDoc} */ + @Override public void compute(int row, int col, IgniteTriFunction f) { + setX(row, col, f.apply(row, col, getX(row, col))); + } } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/DenseLocalOnHeapMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/DenseLocalOnHeapMatrix.java index f95e0cc3feb11..393fff66fcb43 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/DenseLocalOnHeapMatrix.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/DenseLocalOnHeapMatrix.java @@ -18,6 +18,8 @@ package org.apache.ignite.ml.math.impls.matrix; import org.apache.ignite.ml.math.Matrix; +import org.apache.ignite.ml.math.OrderedMatrix; +import org.apache.ignite.ml.math.StorageConstants; import org.apache.ignite.ml.math.Vector; import org.apache.ignite.ml.math.impls.storage.matrix.ArrayMatrixStorage; import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector; @@ -30,7 +32,7 @@ * local, non-distributed execution is satisfactory and on-heap JVM storage is enough * to keep the entire data set. */ -public class DenseLocalOnHeapMatrix extends AbstractMatrix { +public class DenseLocalOnHeapMatrix extends AbstractMatrix implements OrderedMatrix { /** * */ @@ -43,44 +45,89 @@ public DenseLocalOnHeapMatrix() { * @param cols Amount of columns in matrix. */ public DenseLocalOnHeapMatrix(int rows, int cols) { + this(rows, cols, StorageConstants.ROW_STORAGE_MODE); + } + + /** + * @param rows Amount of rows in matrix. + * @param cols Amount of columns in matrix. + * @param acsMode Storage order (row or column-based). + */ + public DenseLocalOnHeapMatrix(int rows, int cols, int acsMode) { assert rows > 0; assert cols > 0; - setStorage(new ArrayMatrixStorage(rows, cols)); + setStorage(new ArrayMatrixStorage(rows, cols, acsMode)); + } + + /** + * @param mtx Backing data array. + * @param acsMode Access mode. + */ + public DenseLocalOnHeapMatrix(double[][] mtx, int acsMode) { + assert mtx != null; + + setStorage(new ArrayMatrixStorage(mtx, acsMode)); } /** * @param mtx Backing data array. */ public DenseLocalOnHeapMatrix(double[][] mtx) { + this(mtx, StorageConstants.ROW_STORAGE_MODE); + } + + /** + * @param mtx Backing data array. + * @param acsMode Access mode. + */ + public DenseLocalOnHeapMatrix(double[] mtx, int rows, int acsMode) { assert mtx != null; - setStorage(new ArrayMatrixStorage(mtx)); + setStorage(new ArrayMatrixStorage(mtx, rows, acsMode)); } /** - * @param orig Original matrix. + * Build new matrix from flat raw array. */ + public DenseLocalOnHeapMatrix(double[] mtx, int rows) { + this(mtx, StorageConstants.ROW_STORAGE_MODE, rows); + } + + /** */ private DenseLocalOnHeapMatrix(DenseLocalOnHeapMatrix orig) { + this(orig, orig.accessMode()); + } + + /** + * @param orig Original matrix. + * @param acsMode Access mode. + */ + private DenseLocalOnHeapMatrix(DenseLocalOnHeapMatrix orig, int acsMode) { assert orig != null; - setStorage(new ArrayMatrixStorage(orig.rowSize(), orig.columnSize())); + setStorage(new ArrayMatrixStorage(orig.rowSize(), orig.columnSize(), acsMode)); assign(orig); } /** {@inheritDoc} */ @Override public Matrix copy() { - return new DenseLocalOnHeapMatrix(this); + return new DenseLocalOnHeapMatrix(this, accessMode()); } /** {@inheritDoc} */ @Override public Matrix like(int rows, int cols) { - return new DenseLocalOnHeapMatrix(rows, cols); + return new DenseLocalOnHeapMatrix(rows, cols, getStorage() != null ? accessMode() : StorageConstants.ROW_STORAGE_MODE); } /** {@inheritDoc} */ @Override public Vector likeVector(int crd) { return new DenseLocalOnHeapVector(crd); } + + /** */ + @Override public int accessMode() { + return ((ArrayMatrixStorage)getStorage()).accessMode(); + } } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseLocalOnHeapMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseLocalOnHeapMatrix.java index d711295bfec32..d0a5937a87b3c 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseLocalOnHeapMatrix.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseLocalOnHeapMatrix.java @@ -17,10 +17,14 @@ package org.apache.ignite.ml.math.impls.matrix; +import it.unimi.dsi.fastutil.ints.Int2ObjectArrayMap; +import it.unimi.dsi.fastutil.ints.IntIterator; +import it.unimi.dsi.fastutil.ints.IntSet; import org.apache.ignite.ml.math.Matrix; import org.apache.ignite.ml.math.MatrixStorage; import org.apache.ignite.ml.math.StorageConstants; import org.apache.ignite.ml.math.Vector; +import org.apache.ignite.ml.math.functions.IgniteTriFunction; import org.apache.ignite.ml.math.impls.storage.matrix.SparseLocalOnHeapMatrixStorage; import org.apache.ignite.ml.math.impls.vector.SparseLocalVector; @@ -61,6 +65,24 @@ private MatrixStorage mkStorage(int rows, int cols) { return new SparseLocalVector(crd, StorageConstants.RANDOM_ACCESS_MODE); } + /** {@inheritDoc} */ + @Override public int nonZeroElements() { + int res = 0; + IntIterator rowIter = indexesMap().keySet().iterator(); + + while (rowIter.hasNext()) { + int row = rowIter.nextInt(); + res += indexesMap().get(row).size(); + } + + return res; + } + + /** */ + public Int2ObjectArrayMap indexesMap() { + return ((SparseLocalOnHeapMatrixStorage)getStorage()).indexesMap(); + } + /** {@inheritDoc} */ @Override public Matrix copy() { Matrix cp = like(rowSize(), columnSize()); @@ -69,4 +91,9 @@ private MatrixStorage mkStorage(int rows, int cols) { return cp; } + + /** {@inheritDoc} */ + @Override public void compute(int row, int col, IgniteTriFunction f) { + ((SparseLocalOnHeapMatrixStorage)getStorage()).compute(row, col, f); + } } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/ArrayMatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/ArrayMatrixStorage.java index 397bf93b80ec3..1f337fdad889a 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/ArrayMatrixStorage.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/ArrayMatrixStorage.java @@ -22,17 +22,24 @@ import java.io.ObjectOutput; import java.util.Arrays; import org.apache.ignite.ml.math.MatrixStorage; +import org.apache.ignite.ml.math.StorageConstants; +import org.apache.ignite.ml.math.functions.IgniteIntIntToIntBiFunction; +import org.apache.ignite.ml.math.util.MatrixUtil; /** * Array based {@link MatrixStorage} implementation. */ public class ArrayMatrixStorage implements MatrixStorage { /** Backing data array. */ - private double[][] data; + private double[] data; /** Amount of rows in the matrix. */ private int rows; /** Amount of columns in the matrix. */ private int cols; + /** Mode specifying if this matrix is row-major or column-major. */ + private int acsMode; + /** Index mapper */ + private IgniteIntIntToIntBiFunction idxMapper; /** * @@ -46,32 +53,62 @@ public ArrayMatrixStorage() { * @param cols Amount of columns in the matrix. */ public ArrayMatrixStorage(int rows, int cols) { + this(rows, cols, StorageConstants.ROW_STORAGE_MODE); + } + + /** */ + public ArrayMatrixStorage(int rows, int cols, int acsMode) { assert rows > 0; assert cols > 0; - this.data = new double[rows][cols]; + this.data = new double[rows * cols]; this.rows = rows; this.cols = cols; + idxMapper = indexMapper(acsMode); + this.acsMode = acsMode; + } + + /** + * @param data Backing data array. + */ + public ArrayMatrixStorage(double[][] data, int acsMode) { + this(MatrixUtil.flatten(data, acsMode), data.length, acsMode); } /** * @param data Backing data array. */ public ArrayMatrixStorage(double[][] data) { + this(MatrixUtil.flatten(data, StorageConstants.ROW_STORAGE_MODE), data.length); + } + + /** + * @param data Backing data array. + */ + public ArrayMatrixStorage(double[] data, int rows, int acsMode) { assert data != null; - assert data[0] != null; + assert data.length % rows == 0; this.data = data; - this.rows = data.length; - this.cols = data[0].length; + this.rows = rows; + this.cols = data.length / rows; + idxMapper = indexMapper(acsMode); + this.acsMode = acsMode; assert rows > 0; assert cols > 0; } + /** + * @param data Backing data array. + */ + public ArrayMatrixStorage(double[] data, int rows) { + this(data, rows, StorageConstants.ROW_STORAGE_MODE); + } + /** {@inheritDoc} */ @Override public double get(int x, int y) { - return data[x][y]; + return data[idxMapper.apply(x, y)]; } /** {@inheritDoc} */ @@ -96,7 +133,7 @@ public ArrayMatrixStorage(double[][] data) { /** {@inheritDoc} */ @Override public void set(int x, int y, double v) { - data[x][y] = v; + data[idxMapper.apply(x, y)] = v; } /** {@inheritDoc} */ @@ -115,14 +152,25 @@ public ArrayMatrixStorage(double[][] data) { } /** {@inheritDoc} */ - @Override public double[][] data() { + @Override public double[] data() { return data; } + /** + * Get the index mapper for given access mode. + * + * @param acsMode Access mode. + */ + private IgniteIntIntToIntBiFunction indexMapper(int acsMode) { + return acsMode == StorageConstants.ROW_STORAGE_MODE ? (r, c) -> r * cols + c : + (r, c) -> c * rows + r; + } + /** {@inheritDoc} */ @Override public void writeExternal(ObjectOutput out) throws IOException { out.writeInt(rows); out.writeInt(cols); + out.writeInt(acsMode); out.writeObject(data); } @@ -131,8 +179,15 @@ public ArrayMatrixStorage(double[][] data) { @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { rows = in.readInt(); cols = in.readInt(); + acsMode = in.readInt(); + idxMapper = indexMapper(acsMode); + + data = (double[])in.readObject(); + } - data = (double[][])in.readObject(); + /** Get the access mode of this storage. */ + public int accessMode() { + return acsMode; } /** {@inheritDoc} */ @@ -141,7 +196,8 @@ public ArrayMatrixStorage(double[][] data) { res += res * 37 + rows; res += res * 37 + cols; - res += res * 37 + Arrays.deepHashCode(data); + res += res * 37 + acsMode; + res += res * 37 + Arrays.hashCode(data); return res; } @@ -156,6 +212,6 @@ public ArrayMatrixStorage(double[][] data) { ArrayMatrixStorage that = (ArrayMatrixStorage)o; - return Arrays.deepEquals(data, that.data); + return acsMode == that.acsMode && Arrays.equals(data, that.data); } } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/DenseOffHeapMatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/DenseOffHeapMatrixStorage.java index 7405a4e080c9f..f58da6581e2c5 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/DenseOffHeapMatrixStorage.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/DenseOffHeapMatrixStorage.java @@ -122,7 +122,7 @@ public DenseOffHeapMatrixStorage(double[][] data) { } /** {@inheritDoc} */ - @Override public double[][] data() { + @Override public double[] data() { return null; } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MatrixDelegateStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MatrixDelegateStorage.java index 1f77d0fc6fb24..f1854793751e6 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MatrixDelegateStorage.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MatrixDelegateStorage.java @@ -150,7 +150,7 @@ public int columnsLength() { } /** {@inheritDoc} */ - @Override public double[][] data() { + @Override public double[] data() { return sto.data(); } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseDistributedMatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseDistributedMatrixStorage.java index 1513502d98ea6..fc7d96999af8e 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseDistributedMatrixStorage.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseDistributedMatrixStorage.java @@ -116,9 +116,7 @@ private IgniteCache, Map> ne // Random cache name. cfg.setName(ML_CACHE_NAME); - IgniteCache, Map> cache = Ignition.localIgnite().getOrCreateCache(cfg); - - return cache; + return Ignition.localIgnite().getOrCreateCache(cfg); } /** diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseLocalOnHeapMatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseLocalOnHeapMatrixStorage.java index b33cb266f5ff8..daf1d4b83097d 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseLocalOnHeapMatrixStorage.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseLocalOnHeapMatrixStorage.java @@ -19,6 +19,8 @@ import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; import it.unimi.dsi.fastutil.ints.Int2DoubleRBTreeMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectArrayMap; +import it.unimi.dsi.fastutil.ints.IntSet; import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; @@ -26,6 +28,7 @@ import java.util.Map; import org.apache.ignite.ml.math.MatrixStorage; import org.apache.ignite.ml.math.StorageConstants; +import org.apache.ignite.ml.math.functions.IgniteTriFunction; /** * Storage for sparse, local, on-heap matrix. @@ -225,4 +228,19 @@ public int getAccessMode() { return rows == that.rows && cols == that.cols && acsMode == that.acsMode && stoMode == that.stoMode && (sto != null ? sto.equals(that.sto) : that.sto == null); } + + /** */ + public void compute(int row, int col, IgniteTriFunction f) { + sto.get(row).compute(col, (c, val) -> f.apply(row, c, val)); + } + + /** */ + public Int2ObjectArrayMap indexesMap() { + Int2ObjectArrayMap res = new Int2ObjectArrayMap<>(); + + for (Integer row : sto.keySet()) + res.put(row.intValue(), (IntSet)sto.get(row).keySet()); + + return res; + } } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java index deef0109a7d99..3323a0739cb55 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java @@ -19,6 +19,7 @@ import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap; import it.unimi.dsi.fastutil.ints.Int2DoubleRBTreeMap; +import it.unimi.dsi.fastutil.ints.IntSet; import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; @@ -178,4 +179,9 @@ else if (sto.containsKey(i)) return res; } + + /** */ + public IntSet indexes() { + return (IntSet)sto.keySet(); + } } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/AbstractReadOnlyVector.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/AbstractReadOnlyVector.java index e48542b60ed16..1de334f33cea7 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/AbstractReadOnlyVector.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/AbstractReadOnlyVector.java @@ -22,6 +22,7 @@ import org.apache.ignite.ml.math.VectorStorage; import org.apache.ignite.ml.math.functions.IgniteBiFunction; import org.apache.ignite.ml.math.functions.IgniteDoubleFunction; +import org.apache.ignite.ml.math.functions.IgniteIntDoubleToDoubleBiFunction; import org.apache.ignite.ml.math.impls.matrix.FunctionMatrix; /** @@ -122,4 +123,9 @@ private Vector logNormalize(double power, double normLen) { return new FunctionVector(size(), (idx) -> Math.log1p(get(idx)) / denominator); } + + /** {@inheritDoc} */ + @Override public void compute(int idx, IgniteIntDoubleToDoubleBiFunction f) { + throw new UnsupportedOperationException(); + } } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/AbstractVector.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/AbstractVector.java index 83ac837488d02..131a61029f540 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/AbstractVector.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/AbstractVector.java @@ -38,6 +38,7 @@ import org.apache.ignite.ml.math.functions.Functions; import org.apache.ignite.ml.math.functions.IgniteBiFunction; import org.apache.ignite.ml.math.functions.IgniteDoubleFunction; +import org.apache.ignite.ml.math.functions.IgniteIntDoubleToDoubleBiFunction; import org.apache.ignite.ml.math.impls.matrix.MatrixView; import org.jetbrains.annotations.NotNull; @@ -904,4 +905,11 @@ protected double dotSelf() { return (sto != null ? sto.equals(that.sto) : that.sto == null); } + + /** {@inheritDoc} */ + @Override public void compute(int idx, IgniteIntDoubleToDoubleBiFunction f) { + storageSet(idx, f.apply(idx, storageGet(idx))); + lenSq = 0.0; + maxElm = minElm = null; + } } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/DelegatingVector.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/DelegatingVector.java index 48fbd0605c554..1df9acc9ddad3 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/DelegatingVector.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/DelegatingVector.java @@ -30,6 +30,7 @@ import org.apache.ignite.ml.math.VectorStorage; import org.apache.ignite.ml.math.functions.IgniteBiFunction; import org.apache.ignite.ml.math.functions.IgniteDoubleFunction; +import org.apache.ignite.ml.math.functions.IgniteIntDoubleToDoubleBiFunction; /** * Convenient class that can be used to add decorations to an existing vector. Subclasses @@ -366,6 +367,11 @@ public Vector getVector() { return guid; } + /** {@inheritDoc} */ + @Override public void compute(int i, IgniteIntDoubleToDoubleBiFunction f) { + dlg.compute(i, f); + } + /** {@inheritDoc} */ @Override public void destroy() { dlg.destroy(); diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseLocalVector.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseLocalVector.java index 9e345bb8ada67..bc1e59da30827 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseLocalVector.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/SparseLocalVector.java @@ -17,7 +17,11 @@ package org.apache.ignite.ml.math.impls.vector; +import it.unimi.dsi.fastutil.ints.IntSet; import java.util.Map; +import java.util.Set; +import java.util.Spliterator; +import java.util.function.Consumer; import org.apache.ignite.ml.math.Matrix; import org.apache.ignite.ml.math.StorageConstants; import org.apache.ignite.ml.math.Vector; @@ -77,4 +81,39 @@ private SparseLocalOnHeapVectorStorage storage() { else return super.times(x); } + + /** Indexes of non-default elements. */ + public IntSet indexes() { + return storage().indexes(); + } + + /** {@inheritDoc} */ + @Override public Spliterator nonZeroSpliterator() { + return new Spliterator() { + /** {@inheritDoc} */ + @Override public boolean tryAdvance(Consumer act) { + Set indexes = storage().indexes(); + + for (Integer index : indexes) + act.accept(storageGet(index)); + + return true; + } + + /** {@inheritDoc} */ + @Override public Spliterator trySplit() { + return null; // No Splitting. + } + + /** {@inheritDoc} */ + @Override public long estimateSize() { + return storage().indexes().size(); + } + + /** {@inheritDoc} */ + @Override public int characteristics() { + return ORDERED | SIZED; + } + }; + } } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java index 752929d4cdd8e..c727e44604c17 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java @@ -20,12 +20,14 @@ import java.util.List; import org.apache.ignite.internal.util.GridArgumentCheck; import org.apache.ignite.ml.math.Matrix; +import org.apache.ignite.ml.math.StorageConstants; import org.apache.ignite.ml.math.Vector; import org.apache.ignite.ml.math.impls.matrix.CacheMatrix; import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix; import org.apache.ignite.ml.math.impls.matrix.MatrixView; import org.apache.ignite.ml.math.impls.matrix.PivotedMatrixView; import org.apache.ignite.ml.math.impls.matrix.RandomMatrix; +import org.apache.ignite.ml.math.impls.matrix.SparseLocalOnHeapMatrix; import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector; /** @@ -115,6 +117,18 @@ public static Matrix copy(Matrix matrix) { return matrix.copy(); } + /** */ + public static DenseLocalOnHeapMatrix asDense(SparseLocalOnHeapMatrix m, int acsMode) { + DenseLocalOnHeapMatrix res = new DenseLocalOnHeapMatrix(m.rowSize(), m.columnSize(), acsMode); + + for (Integer row : m.indexesMap().keySet()) { + for (Integer col : m.indexesMap().get(row)) + res.set(row, col, m.get(row, col)); + } + + return res; + } + /** */ private static boolean isCopyLikeSupport(Matrix matrix) { return matrix instanceof RandomMatrix || matrix instanceof MatrixView || matrix instanceof CacheMatrix || @@ -152,4 +166,38 @@ public static DenseLocalOnHeapVector localCopyOf(Vector vec) { return res; } + + /** */ + public static double[][] unflatten(double[] fArr, int colsCnt) { + int rowsCnt = fArr.length / colsCnt; + + double[][] res = new double[rowsCnt][colsCnt]; + + for (int i = 0; i < rowsCnt; i++) + for (int j = 0; j < colsCnt; j++) + res[i][j] = fArr[i * colsCnt + j]; + + return res; + } + + /** */ + public static double[] flatten(double[][] arr, int acsMode) { + assert arr != null; + assert arr[0] != null; + + int size = arr.length * arr[0].length; + int rows = acsMode == StorageConstants.ROW_STORAGE_MODE ? arr.length : arr[0].length; + int cols = size / rows; + + double[] res = new double[size]; + + int iLim = acsMode == StorageConstants.ROW_STORAGE_MODE ? rows : cols; + int jLim = acsMode == StorageConstants.ROW_STORAGE_MODE ? cols : rows; + + for (int i = 0; i < iLim; i++) + for (int j = 0; j < jLim; j++) + res[i * jLim + j] = arr[i][j]; + + return res; + } } diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTest.java index cdc2651393f9f..a59b7f922343c 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTest.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTest.java @@ -130,8 +130,8 @@ public void testClusterizationOnDatasetWithObviousStructure() throws IOException Vector[] mc = new Vector[centersCnt]; Arrays.fill(mc, VectorUtils.zeroes(2)); - int centIndex = 0; - int totalCount = 0; + int centIdx = 0; + int totalCnt = 0; List massCenters = new ArrayList<>(); @@ -140,12 +140,12 @@ public void testClusterizationOnDatasetWithObviousStructure() throws IOException DenseLocalOnHeapVector pnt = (DenseLocalOnHeapVector)new DenseLocalOnHeapVector(2).assign(centers.get(count)); // pertrubate point on random value. pnt.map(val -> val + rnd.nextDouble() * squareSideLen / 100); - mc[centIndex] = mc[centIndex].plus(pnt); - points.assignRow(permutation.get(totalCount), pnt); - totalCount++; + mc[centIdx] = mc[centIdx].plus(pnt); + points.assignRow(permutation.get(totalCnt), pnt); + totalCnt++; } - massCenters.add(mc[centIndex].times(1 / (double)count)); - centIndex++; + massCenters.add(mc[centIdx].times(1 / (double)count)); + centIdx++; } EuclideanDistance dist = new EuclideanDistance(); @@ -169,6 +169,7 @@ private static class OrderedNodesComparator implements Comparator { /** */ List orderedNodes; + /** */ public OrderedNodesComparator(Vector[] orderedNodes, DistanceMeasure measure) { this.orderedNodes = Arrays.asList(orderedNodes); this.measure = measure; diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/BlasTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/BlasTest.java new file mode 100644 index 0000000000000..00bce47a122f2 --- /dev/null +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/BlasTest.java @@ -0,0 +1,357 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math; + +import java.util.function.BiPredicate; +import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix; +import org.apache.ignite.ml.math.impls.matrix.SparseLocalOnHeapMatrix; +import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector; +import org.apache.ignite.ml.math.impls.vector.SparseLocalVector; +import org.apache.ignite.ml.math.util.MatrixUtil; +import org.junit.Assert; +import org.junit.Test; + +/** Tests for BLAS operations (all operations are only available for local matrices and vectors). */ +public class BlasTest { + /** Test 'axpy' operation for two array-based vectors. */ + @Test + public void testAxpyArrayArray() { + Vector y = new DenseLocalOnHeapVector(new double[] {1.0, 2.0}); + double a = 2.0; + Vector x = new DenseLocalOnHeapVector(new double[] {1.0, 2.0}); + + Vector exp = x.times(a).plus(y); + Blas.axpy(a, x, y); + + Assert.assertEquals(y, exp); + } + + /** Test 'axpy' operation for sparse vector and array-based vector. */ + @Test + public void testAxpySparseArray() { + DenseLocalOnHeapVector y = new DenseLocalOnHeapVector(new double[] {1.0, 2.0}); + double a = 2.0; + SparseLocalVector x = sparseFromArray(new double[] {1.0, 2.0}); + + SparseLocalVector exp = (SparseLocalVector)x.times(a).plus(y); + Blas.axpy(a, x, y); + + Assert.assertTrue(elementsEqual(exp, y)); + } + + /** Test 'dot' operation. */ + @Test + public void testDot() { + DenseLocalOnHeapVector v1 = new DenseLocalOnHeapVector(new double[] {1.0, 1.0}); + DenseLocalOnHeapVector v2 = new DenseLocalOnHeapVector(new double[] {2.0, 2.0}); + + Assert.assertEquals(Blas.dot(v1, v2), v1.dot(v2), 0.0); + } + + /** Test 'scal' operation for dense matrix. */ + @Test + public void testScalDense() { + double[] data = new double[] {1.0, 1.0}; + double alpha = 2.0; + + DenseLocalOnHeapVector v = new DenseLocalOnHeapVector(data); + Vector exp = new DenseLocalOnHeapVector(data, true).times(alpha); + Blas.scal(alpha, v); + + Assert.assertEquals(v, exp); + } + + /** Test 'scal' operation for sparse matrix. */ + @Test + public void testScalSparse() { + double[] data = new double[] {1.0, 1.0}; + double alpha = 2.0; + + SparseLocalVector v = sparseFromArray(data); + Vector exp = sparseFromArray(data).times(alpha); + + Blas.scal(alpha, v); + + Assert.assertEquals(v, exp); + } + + /** Test 'spr' operation for dense vector v and dense matrix A. */ + @Test + public void testSprDenseDense() { + double alpha = 3.0; + + DenseLocalOnHeapVector v = new DenseLocalOnHeapVector(new double[] {1.0, 2.0}); + DenseLocalOnHeapVector u = new DenseLocalOnHeapVector(new double[] {3.0, 13.0, 20.0, 0.0}); + + // m is alpha * v * v^t + DenseLocalOnHeapMatrix m = (DenseLocalOnHeapMatrix)new DenseLocalOnHeapMatrix(new double[][] {{1.0, 0.0}, + {2.0, 4.0}}, StorageConstants.COLUMN_STORAGE_MODE).times(alpha); + DenseLocalOnHeapMatrix a = new DenseLocalOnHeapMatrix(new double[][] {{3.0, 0.0}, {13.0, 20.0}}, + StorageConstants.COLUMN_STORAGE_MODE); + + //m := alpha * v * v.t + A + Blas.spr(alpha, v, u); + + DenseLocalOnHeapMatrix mu = fromVector(u, a.rowSize(), StorageConstants.COLUMN_STORAGE_MODE, (i, j) -> i >= j); + Assert.assertEquals(m.plus(a), mu); + } + + /** Test 'spr' operation for sparse vector v (sparse in representation, dense in fact) and dense matrix A. */ + @Test + public void testSprSparseDense1() { + double alpha = 3.0; + + SparseLocalVector v = sparseFromArray(new double[] {1.0, 2.0}); + DenseLocalOnHeapVector u = new DenseLocalOnHeapVector(new double[] {3.0, 13.0, 20.0, 0.0}); + + DenseLocalOnHeapMatrix a = new DenseLocalOnHeapMatrix(new double[][] {{3.0, 0.0}, {13.0, 20.0}}, + StorageConstants.COLUMN_STORAGE_MODE); + DenseLocalOnHeapMatrix exp = (DenseLocalOnHeapMatrix)new DenseLocalOnHeapMatrix(new double[][] {{1.0, 0.0}, + {2.0, 4.0}}, StorageConstants.COLUMN_STORAGE_MODE).times(alpha).plus(a); + + //m := alpha * v * v.t + A + Blas.spr(alpha, v, u); + DenseLocalOnHeapMatrix mu = fromVector(u, a.rowSize(), StorageConstants.COLUMN_STORAGE_MODE, (i, j) -> i >= j); + Assert.assertEquals(exp, mu); + } + + /** Test 'spr' operation for sparse vector v (sparse in representation, sparse in fact) and dense matrix A. */ + @Test + public void testSprSparseDense2() { + double alpha = 3.0; + + SparseLocalVector v = new SparseLocalVector(2, StorageConstants.RANDOM_ACCESS_MODE); + v.set(0, 1); + + DenseLocalOnHeapVector u = new DenseLocalOnHeapVector(new double[] {3.0, 13.0, 20.0, 0.0}); + + // m is alpha * v * v^t + DenseLocalOnHeapMatrix m = (DenseLocalOnHeapMatrix)new DenseLocalOnHeapMatrix(new double[][] {{1.0, 0.0}, + {0.0, 0.0}}, StorageConstants.COLUMN_STORAGE_MODE).times(alpha); + DenseLocalOnHeapMatrix a = new DenseLocalOnHeapMatrix(new double[][] {{3.0, 0.0}, {13.0, 20.0}}, + StorageConstants.COLUMN_STORAGE_MODE); + + //m := alpha * v * v.t + A + Blas.spr(alpha, v, u); + DenseLocalOnHeapMatrix mu = fromVector(u, a.rowSize(), StorageConstants.COLUMN_STORAGE_MODE, (i, j) -> i >= j); + Assert.assertEquals(m.plus(a), mu); + } + + /** Tests 'syr' operation for dense vector x and dense matrix A. */ + @Test + public void testSyrDenseDense() { + double alpha = 2.0; + DenseLocalOnHeapVector x = new DenseLocalOnHeapVector(new double[] {1.0, 2.0}); + DenseLocalOnHeapMatrix a = new DenseLocalOnHeapMatrix(new double[][] {{10.0, 20.0}, {20.0, 10.0}}); + + // alpha * x * x^T + A + DenseLocalOnHeapMatrix exp = (DenseLocalOnHeapMatrix)new DenseLocalOnHeapMatrix(new double[][] {{1.0, 2.0}, + {2.0, 4.0}}).times(alpha).plus(a); + + Blas.syr(alpha, x, a); + + Assert.assertEquals(exp, a); + } + + /** Tests 'gemm' operation for dense matrix A, dense matrix B and dense matrix C. */ + @Test + public void testGemmDenseDenseDense() { + // C := alpha * A * B + beta * C + double alpha = 2.0; + DenseLocalOnHeapMatrix a = new DenseLocalOnHeapMatrix(new double[][] {{10.0, 11.0}, {0.0, 1.0}}); + DenseLocalOnHeapMatrix b = new DenseLocalOnHeapMatrix(new double[][] {{1.0, 0.0}, {0.0, 1.0}}); + double beta = 3.0; + DenseLocalOnHeapMatrix c = new DenseLocalOnHeapMatrix(new double[][] {{1.0, 2.0}, {2.0, 3.0}}); + + DenseLocalOnHeapMatrix exp = (DenseLocalOnHeapMatrix)a.times(b).times(alpha).plus(c.times(beta)); + + Blas.gemm(alpha, a, b, beta, c); + Assert.assertEquals(exp, c); + } + + /** Tests 'gemm' operation for sparse matrix A, dense matrix B and dense matrix C. */ + @Test + public void testGemmSparseDenseDense() { + // C := alpha * A * B + beta * C + double alpha = 2.0; + SparseLocalOnHeapMatrix a = sparseFromArray(new double[][] {{10.0, 11.0}, {0.0, 1.0}}, 2); + DenseLocalOnHeapMatrix b = new DenseLocalOnHeapMatrix(new double[][] {{1.0, 0.0}, {0.0, 1.0}}); + double beta = 3.0; + DenseLocalOnHeapMatrix c = new DenseLocalOnHeapMatrix(new double[][] {{1.0, 2.0}, {2.0, 3.0}}); + + DenseLocalOnHeapMatrix exp = MatrixUtil.asDense((SparseLocalOnHeapMatrix)a.times(b).times(alpha).plus(c.times(beta)), + StorageConstants.ROW_STORAGE_MODE); + + Blas.gemm(alpha, a, b, beta, c); + + Assert.assertEquals(exp, c); + } + + /** Tests 'gemv' operation for dense matrix A, dense vector x and dense vector y. */ + @Test + public void testGemvSparseDenseDense() { + // y := alpha * A * x + beta * y + double alpha = 3.0; + SparseLocalOnHeapMatrix a = sparseFromArray(new double[][] {{10.0, 11.0}, {0.0, 1.0}}, 2); + DenseLocalOnHeapVector x = new DenseLocalOnHeapVector(new double[] {1.0, 2.0}); + double beta = 2.0; + DenseLocalOnHeapVector y = new DenseLocalOnHeapVector(new double[] {3.0, 4.0}); + + DenseLocalOnHeapVector exp = (DenseLocalOnHeapVector)y.times(beta).plus(a.times(x).times(alpha)); + + Blas.gemv(alpha, a, x, beta, y); + + Assert.assertEquals(exp, y); + } + + /** Tests 'gemv' operation for dense matrix A, sparse vector x and dense vector y. */ + @Test + public void testGemvDenseSparseDense() { + // y := alpha * A * x + beta * y + double alpha = 3.0; + SparseLocalOnHeapMatrix a = sparseFromArray(new double[][] {{10.0, 11.0}, {0.0, 1.0}}, 2); + SparseLocalVector x = sparseFromArray(new double[] {1.0, 2.0}); + double beta = 2.0; + DenseLocalOnHeapVector y = new DenseLocalOnHeapVector(new double[] {3.0, 4.0}); + + DenseLocalOnHeapVector exp = (DenseLocalOnHeapVector)y.times(beta).plus(a.times(x).times(alpha)); + + Blas.gemv(alpha, a, x, beta, y); + + Assert.assertEquals(exp, y); + } + + /** Tests 'gemv' operation for sparse matrix A, sparse vector x and dense vector y. */ + @Test + public void testGemvSparseSparseDense() { + // y := alpha * A * x + beta * y + double alpha = 3.0; + DenseLocalOnHeapMatrix a = new DenseLocalOnHeapMatrix(new double[][] {{10.0, 11.0}, {0.0, 1.0}}, 2); + SparseLocalVector x = sparseFromArray(new double[] {1.0, 2.0}); + double beta = 2.0; + DenseLocalOnHeapVector y = new DenseLocalOnHeapVector(new double[] {3.0, 4.0}); + + DenseLocalOnHeapVector exp = (DenseLocalOnHeapVector)y.times(beta).plus(a.times(x).times(alpha)); + + Blas.gemv(alpha, a, x, beta, y); + + Assert.assertEquals(exp, y); + } + + /** Tests 'gemv' operation for dense matrix A, dense vector x and dense vector y. */ + @Test + public void testGemvDenseDenseDense() { + // y := alpha * A * x + beta * y + double alpha = 3.0; + DenseLocalOnHeapMatrix a = new DenseLocalOnHeapMatrix(new double[][] {{10.0, 11.0}, {0.0, 1.0}}, 2); + DenseLocalOnHeapVector x = new DenseLocalOnHeapVector(new double[] {1.0, 2.0}); + double beta = 2.0; + DenseLocalOnHeapVector y = new DenseLocalOnHeapVector(new double[] {3.0, 4.0}); + + DenseLocalOnHeapVector exp = (DenseLocalOnHeapVector)y.times(beta).plus(a.times(x).times(alpha)); + + Blas.gemv(alpha, a, x, beta, y); + + Assert.assertEquals(exp, y); + } + + /** + * Create a sparse vector from array. + * + * @param arr Array with vector elements. + * @return sparse local on-heap vector. + */ + private static SparseLocalVector sparseFromArray(double[] arr) { + SparseLocalVector res = new SparseLocalVector(2, StorageConstants.RANDOM_ACCESS_MODE); + + for (int i = 0; i < arr.length; i++) + res.setX(i, arr[i]); + + return res; + } + + /** + * Create a sparse matrix from array. + * + * @param arr Array with matrix elements. + * @param rows Number of rows in target matrix. + * @return sparse local on-heap matrix. + */ + private static SparseLocalOnHeapMatrix sparseFromArray(double[][] arr, int rows) { + int cols = arr[0].length; + SparseLocalOnHeapMatrix res = new SparseLocalOnHeapMatrix(rows, cols); + + for (int i = 0; i < rows; i++) + for (int j = 0; j < cols; j++) + res.set(i, j, arr[i][j]); + + return res; + } + + /** + * Checks if two vectors have equal elements. + * + * @param a Matrix a. + * @param b Vector b + * @return true if vectors are equal element-wise, false otherwise. + */ + private static boolean elementsEqual(Vector a, Vector b) { + int n = b.size(); + + for (int i = 0; i < n; i++) + if (a.get(i) != b.get(i)) + return false; + + return true; + } + + /** + * Creates dense local on-heap matrix from vector v entities filtered by bipredicate p. + * + * @param v Vector with entities for matrix to be created. + * @param rows Rows number in the target matrix. + * @param acsMode column or row major mode. + * @param p bipredicate to filter entities by. + * @return dense local on-heap matrix. + */ + private static DenseLocalOnHeapMatrix fromVector(DenseLocalOnHeapVector v, int rows, int acsMode, + BiPredicate p) { + double[] data = v.getStorage().data(); + int cols = data.length / rows; + double[] d = new double[data.length]; + + int iLim = acsMode == StorageConstants.ROW_STORAGE_MODE ? rows : cols; + int jLim = acsMode == StorageConstants.ROW_STORAGE_MODE ? cols : rows; + + int shift = 0; + + for (int i = 0; i < iLim; i++) + for (int j = 0; j < jLim; j++) { + int ind = i * jLim + j; + + if (!p.test(i, j)) { + shift++; + d[ind] = 0.0; + continue; + } + d[ind] = data[ind - shift]; + } + + return new DenseLocalOnHeapMatrix(d, rows, acsMode); + } +} \ No newline at end of file diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplMainTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplMainTestSuite.java index 4d245b4e35eb1..974b7bbc3b53d 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplMainTestSuite.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplMainTestSuite.java @@ -27,7 +27,8 @@ @Suite.SuiteClasses({ MathImplLocalTestSuite.class, MathImplDistributedTestSuite.class, - TracerTest.class + TracerTest.class, + BlasTest.class }) public class MathImplMainTestSuite { // No-op. diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/MatrixViewConstructorTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/MatrixViewConstructorTest.java index 82564cbb27f9b..3e9cdfe3fffc9 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/MatrixViewConstructorTest.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/MatrixViewConstructorTest.java @@ -108,7 +108,7 @@ public void attributeTest() { assertEquals(m.isDense(), delegateStorage.isDense()); assertEquals(m.isArrayBased(), delegateStorage.isArrayBased()); - assertArrayEquals(m.getStorage().data(), delegateStorage.data()); + assertArrayEquals(m.getStorage().data(), delegateStorage.data(), 0.0); } } } diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/storage/matrix/MatrixArrayStorageTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/storage/matrix/MatrixArrayStorageTest.java index 569ed570325fc..3395422eefffb 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/storage/matrix/MatrixArrayStorageTest.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/storage/matrix/MatrixArrayStorageTest.java @@ -54,10 +54,10 @@ public void isArrayBased() throws Exception { /** */ @Test public void data() throws Exception { - double[][] data = storage.data(); + double[] data = storage.data(); assertNotNull(MathTestConstants.NULL_VAL, data); - assertTrue(MathTestConstants.UNEXPECTED_VAL, data.length == MathTestConstants.STORAGE_SIZE); - assertTrue(MathTestConstants.UNEXPECTED_VAL, data[0].length == MathTestConstants.STORAGE_SIZE); + assertTrue(MathTestConstants.UNEXPECTED_VAL, data.length == MathTestConstants.STORAGE_SIZE * + MathTestConstants.STORAGE_SIZE); } } From 85f17027d0ca7cfa6dfaa3711ad8b4a7a6eb1e79 Mon Sep 17 00:00:00 2001 From: Aleksandr_Meterko Date: Wed, 19 Jul 2017 19:55:15 +0700 Subject: [PATCH 027/547] IGNITE-5781 Visor throws ClassCastException if cache store implementation is other than CacheJdbcPojoStore Signed-off-by: Konstantin Boudnik (cherry picked from commit 02e2507) --- .../apache/ignite/internal/visor/cache/VisorCacheJdbcType.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheJdbcType.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheJdbcType.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheJdbcType.java old mode 100644 new mode 100755 index e50402cd1d586..a03096fc183e6 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheJdbcType.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheJdbcType.java @@ -64,7 +64,7 @@ public class VisorCacheJdbcType extends VisorDataTransferObject { public static List list(Factory factory) { List res = new ArrayList<>(); - if (factory != null || factory instanceof CacheJdbcPojoStoreFactory) { + if (factory instanceof CacheJdbcPojoStoreFactory) { CacheJdbcPojoStoreFactory jdbcFactory = (CacheJdbcPojoStoreFactory) factory; JdbcType[] jdbcTypes = jdbcFactory.getTypes(); From 7915fd88b1f3e399777bbc46f4e5625b68fb90c9 Mon Sep 17 00:00:00 2001 From: Jokser Date: Wed, 26 Jul 2017 12:08:03 +0300 Subject: [PATCH 028/547] IGNITE-5830 - Introduce cache start and stop order during cluster activation --- .../processors/cache/ClusterCachesInfo.java | 135 ++++++++++++------ .../processors/cache/ExchangeActions.java | 4 +- .../processors/cache/GridCacheProcessor.java | 29 ++-- 3 files changed, 106 insertions(+), 62 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java index 949bc1989887f..1a05b96917181 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -58,6 +59,7 @@ import org.apache.ignite.plugin.CachePluginProvider; import org.apache.ignite.plugin.PluginProvider; import org.apache.ignite.spi.discovery.DiscoveryDataBag; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import static org.apache.ignite.cache.CacheMode.LOCAL; @@ -97,10 +99,10 @@ class ClusterCachesInfo { private GridData gridData; /** */ - private List> locJoinStartCaches; + private List> locJoinStartCaches = Collections.emptyList(); /** */ - private Map> locCfgsForActivation; + private Map> locCfgsForActivation = Collections.emptyMap(); /** */ private Map clientReconnectReqs; @@ -111,7 +113,7 @@ class ClusterCachesInfo { /** * @param ctx Context. */ - ClusterCachesInfo(GridKernalContext ctx) { + public ClusterCachesInfo(GridKernalContext ctx) { this.ctx = ctx; log = ctx.log(getClass()); @@ -121,7 +123,7 @@ class ClusterCachesInfo { * @param joinDiscoData Information about configured caches and templates. * @throws IgniteCheckedException If configuration validation failed. */ - void onStart(CacheJoinNodeDiscoveryData joinDiscoData) throws IgniteCheckedException { + public void onStart(CacheJoinNodeDiscoveryData joinDiscoData) throws IgniteCheckedException { this.joinDiscoData = joinDiscoData; Map grpCfgs = new HashMap<>(); @@ -159,7 +161,7 @@ private int cacheGroupId(String cacheName, @Nullable String grpName) { * @param checkConsistency {@code True} if need check cache configurations consistency. * @throws IgniteCheckedException If failed. */ - void onKernalStart(boolean checkConsistency) throws IgniteCheckedException { + public void onKernalStart(boolean checkConsistency) throws IgniteCheckedException { if (gridData != null && gridData.conflictErr != null) throw new IgniteCheckedException(gridData.conflictErr); @@ -330,7 +332,7 @@ private void checkCache(CacheJoinNodeDiscoveryData.CacheInfo locInfo, CacheData * @param msg Message. * @param node Node sent message. */ - void onClientCacheChange(ClientCacheChangeDiscoveryMessage msg, ClusterNode node) { + public void onClientCacheChange(ClientCacheChangeDiscoveryMessage msg, ClusterNode node) { Map startedCaches = msg.startedCaches(); if (startedCaches != null) { @@ -359,12 +361,13 @@ void onClientCacheChange(ClientCacheChangeDiscoveryMessage msg, ClusterNode node } } } + /** * @param batch Cache change request. * @param topVer Topology version. * @return {@code True} if minor topology version should be increased. */ - boolean onCacheChangeRequested(DynamicCacheChangeBatch batch, AffinityTopologyVersion topVer) { + public boolean onCacheChangeRequested(DynamicCacheChangeBatch batch, AffinityTopologyVersion topVer) { DiscoveryDataClusterState state = ctx.state().clusterState(); if (state.active() && !state.transition()) { @@ -779,30 +782,28 @@ private Serializable joinDiscoveryData() { * * @return Caches to be started when this node starts. */ - List> cachesToStartOnLocalJoin() { + @NotNull public List> cachesToStartOnLocalJoin() { if (ctx.isDaemon()) return Collections.emptyList(); - assert locJoinStartCaches != null; - - List> locJoinStartCaches = this.locJoinStartCaches; + List> result = locJoinStartCaches; - this.locJoinStartCaches = null; + locJoinStartCaches = Collections.emptyList(); - return locJoinStartCaches; + return result; } /** * @param joinedNodeId Joined node ID. * @return New caches received from joined node. */ - List cachesReceivedFromJoin(UUID joinedNodeId) { + @NotNull public List cachesReceivedFromJoin(UUID joinedNodeId) { assert joinedNodeId != null; List started = null; if (!ctx.isDaemon()) { - for (DynamicCacheDescriptor desc : registeredCaches.values()) { + for (DynamicCacheDescriptor desc : orderedCaches(CacheComparators.DIRECT)) { if (desc.staticallyConfigured()) { assert desc.receivedFrom() != null : desc; @@ -826,7 +827,7 @@ List cachesReceivedFromJoin(UUID joinedNodeId) { * @param node Event node. * @param topVer Topology version. */ - void onDiscoveryEvent(int type, ClusterNode node, AffinityTopologyVersion topVer) { + public void onDiscoveryEvent(int type, ClusterNode node, AffinityTopologyVersion topVer) { if (type == EVT_NODE_JOINED && !ctx.isDaemon()) { for (CacheGroupDescriptor desc : registeredCacheGrps.values()) { if (node.id().equals(desc.receivedFrom())) @@ -856,7 +857,7 @@ void onDiscoveryEvent(int type, ClusterNode node, AffinityTopologyVersion topVer /** * @param dataBag Discovery data bag. */ - void collectGridNodeData(DiscoveryDataBag dataBag) { + public void collectGridNodeData(DiscoveryDataBag dataBag) { if (ctx.isDaemon()) return; @@ -931,7 +932,7 @@ private CacheNodeCommonDiscoveryData collectCommonDiscoveryData() { /** * @param data Discovery data. */ - void onGridDataReceived(DiscoveryDataBag.GridDiscoveryData data) { + public void onGridDataReceived(DiscoveryDataBag.GridDiscoveryData data) { if (ctx.isDaemon() || data.commonData() == null) return; @@ -1045,6 +1046,9 @@ void onGridDataReceived(DiscoveryDataBag.GridDiscoveryData data) { } /** + * Initialize collection with caches to be start: + * {@code locJoinStartCaches} or {@code locCfgsForActivation} if cluster is inactive. + * * @param firstNode {@code True} if first node in cluster starts. */ private void initStartCachesForLocalJoin(boolean firstNode) { @@ -1062,7 +1066,7 @@ private void initStartCachesForLocalJoin(boolean firstNode) { boolean active = ctx.state().clusterState().active(); - for (DynamicCacheDescriptor desc : registeredCaches.values()) { + for (DynamicCacheDescriptor desc : orderedCaches(CacheComparators.DIRECT)) { if (firstNode && !joinDiscoData.caches().containsKey(desc.cacheName())) continue; @@ -1096,13 +1100,8 @@ private void initStartCachesForLocalJoin(boolean firstNode) { if (locCfg != null || joinDiscoData.startCaches() || CU.affinityNode(ctx.discovery().localNode(), desc.groupDescriptor().config().getNodeFilter())) { - if (active) { - // Move system and internal caches first. - if (desc.cacheType().userCache()) - locJoinStartCaches.add(new T2<>(desc, nearCfg)); - else - locJoinStartCaches.add(0, new T2<>(desc, nearCfg)); - } + if (active) + locJoinStartCaches.add(new T2<>(desc, nearCfg)); else locCfgsForActivation.put(desc.cacheName(), new T2<>(desc.cacheConfiguration(), nearCfg)); } @@ -1113,7 +1112,7 @@ private void initStartCachesForLocalJoin(boolean firstNode) { /** * @param msg Message. */ - void onStateChangeFinish(ChangeGlobalStateFinishMessage msg) { + public void onStateChangeFinish(ChangeGlobalStateFinishMessage msg) { if (joinOnTransition) { initStartCachesForLocalJoin(false); @@ -1127,17 +1126,14 @@ void onStateChangeFinish(ChangeGlobalStateFinishMessage msg) { * @return Exchange action. * @throws IgniteCheckedException If configuration validation failed. */ - ExchangeActions onStateChangeRequest(ChangeGlobalStateMessage msg, AffinityTopologyVersion topVer) + public ExchangeActions onStateChangeRequest(ChangeGlobalStateMessage msg, AffinityTopologyVersion topVer) throws IgniteCheckedException { ExchangeActions exchangeActions = new ExchangeActions(); if (msg.activate()) { - for (DynamicCacheDescriptor desc : registeredCaches.values()) { + for (DynamicCacheDescriptor desc : orderedCaches(CacheComparators.DIRECT)) { desc.startTopologyVersion(topVer); - T2 locCfg = !F.isEmpty(locCfgsForActivation) ? - locCfgsForActivation.get(desc.cacheName()) : null; - DynamicCacheChangeRequest req = new DynamicCacheChangeRequest(msg.requestId(), desc.cacheName(), msg.initiatorNodeId()); @@ -1145,6 +1141,8 @@ ExchangeActions onStateChangeRequest(ChangeGlobalStateMessage msg, AffinityTopol req.startCacheConfiguration(desc.cacheConfiguration()); req.cacheType(desc.cacheType()); + T2 locCfg = locCfgsForActivation.get(desc.cacheName()); + if (locCfg != null) { if (locCfg.get1() != null) req.startCacheConfiguration(locCfg.get1()); @@ -1199,7 +1197,7 @@ ExchangeActions onStateChangeRequest(ChangeGlobalStateMessage msg, AffinityTopol else { locCfgsForActivation = new HashMap<>(); - for (DynamicCacheDescriptor desc : registeredCaches.values()) { + for (DynamicCacheDescriptor desc : orderedCaches(CacheComparators.REVERSE)) { DynamicCacheChangeRequest req = DynamicCacheChangeRequest.stopRequest(ctx, desc.cacheName(), desc.sql(), @@ -1221,7 +1219,7 @@ ExchangeActions onStateChangeRequest(ChangeGlobalStateMessage msg, AffinityTopol /** * @param data Joining node data. */ - void onJoiningNodeDataReceived(DiscoveryDataBag.JoiningNodeDiscoveryData data) { + public void onJoiningNodeDataReceived(DiscoveryDataBag.JoiningNodeDiscoveryData data) { if (data.hasJoiningNodeData()) { Serializable joiningNodeData = data.joiningNodeData(); @@ -1264,8 +1262,10 @@ private void processClientReconnectData(CacheClientReconnectDiscoveryData client } /** + * Checks cache configuration on conflict with already registered caches and cache groups. + * * @param cfg Cache configuration. - * @return {@code True} if validation passed. + * @return {@code null} if validation passed, error message in other case. */ private String checkCacheConflict(CacheConfiguration cfg) { int cacheId = CU.cacheId(cfg.getName()); @@ -1479,18 +1479,11 @@ private CacheGroupDescriptor registerCacheGroup( return grpDesc; } - /** - * @return Registered cache groups. - */ - ConcurrentMap registeredCacheGroups() { - return registeredCacheGrps; - } - /** * @param ccfg Cache configuration to start. * @throws IgniteCheckedException If failed. */ - void validateStartCacheConfiguration(CacheConfiguration ccfg) throws IgniteCheckedException { + public void validateStartCacheConfiguration(CacheConfiguration ccfg) throws IgniteCheckedException { if (ccfg.getGroupName() != null) { CacheGroupDescriptor grpDesc = cacheGroupByName(ccfg.getGroupName()); @@ -1562,10 +1555,30 @@ ConcurrentMap registeredTemplates() { return registeredTemplates; } + /** + * @return Registered cache groups. + */ + ConcurrentMap registeredCacheGroups() { + return registeredCacheGrps; + } + + /** + * Returns registered cache descriptors ordered by {@code comparator} + * @param comparator Comparator (DIRECT, REVERSE or custom) to order cache descriptors. + * @return Ordered by comparator cache descriptors. + */ + private Collection orderedCaches(Comparator comparator) { + List ordered = new ArrayList<>(); + ordered.addAll(registeredCaches.values()); + + Collections.sort(ordered, comparator); + return ordered; + } + /** * */ - void onDisconnect() { + public void onDisconnected() { cachesOnDisconnect = new CachesOnDisconnect( ctx.state().clusterState(), new HashMap<>(registeredCacheGrps), @@ -1583,7 +1596,7 @@ void onDisconnect() { * @param transition {@code True} if reconnected while state transition in progress. * @return Information about stopped caches and cache groups. */ - ClusterCachesReconnectResult onReconnected(boolean active, boolean transition) { + public ClusterCachesReconnectResult onReconnected(boolean active, boolean transition) { assert disconnectedState(); Set stoppedCaches = new HashSet<>(); @@ -1684,6 +1697,38 @@ private boolean surviveReconnect(String cacheName) { return CU.isUtilityCache(cacheName); } + /** + * Holds direct comparator (first system caches) and reverse comparator (first user caches). + * Use DIRECT comparator for ordering cache start operations. + * Use REVERSE comparator for ordering cache stop operations. + */ + private static class CacheComparators { + /** + * DIRECT comparator for cache descriptors (first system caches). + */ + static Comparator DIRECT = new Comparator() { + @Override + public int compare(DynamicCacheDescriptor o1, DynamicCacheDescriptor o2) { + if (!o1.cacheType().userCache()) + return -1; + if (!o2.cacheType().userCache()) + return 1; + + return o1.cacheId().compareTo(o2.cacheId()); + } + }; + + /** + * REVERSE comparator for cache descriptors (first user caches). + */ + static Comparator REVERSE = new Comparator() { + @Override + public int compare(DynamicCacheDescriptor o1, DynamicCacheDescriptor o2) { + return -DIRECT.compare(o1, o2); + } + }; + } + /** * */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java index 1cc6438471d21..91ad003f5c9bc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java @@ -20,8 +20,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -224,7 +224,7 @@ private Map add(Map map, assert desc != null; if (map == null) - map = new HashMap<>(); + map = new LinkedHashMap<>(); CacheActionData old = map.put(req.cacheName(), new CacheActionData(req, desc)); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 5b709b3bfb761..9902a92b78abe 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -146,6 +146,7 @@ import org.apache.ignite.spi.discovery.DiscoveryDataBag; import org.apache.ignite.spi.discovery.DiscoveryDataBag.GridDiscoveryData; import org.apache.ignite.spi.discovery.DiscoveryDataBag.JoiningNodeDiscoveryData; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import static org.apache.ignite.IgniteSystemProperties.IGNITE_CACHE_REMOVED_ENTRIES_TTL; @@ -1046,7 +1047,7 @@ public void onKernalStopCaches(boolean cancel) { sharedCtx.onDisconnected(reconnectFut); - cachesInfo.onDisconnect(); + cachesInfo.onDisconnected(); } /** @@ -1733,7 +1734,7 @@ public CacheMode cacheMode(String cacheName) { /** * @return Caches to be started when this node starts. */ - public List> cachesToStartOnLocalJoin() { + @NotNull public List> cachesToStartOnLocalJoin() { return cachesInfo.cachesToStartOnLocalJoin(); } @@ -1771,22 +1772,20 @@ public Collection startReceivedCaches(UUID nodeId, Affin throws IgniteCheckedException { List started = cachesInfo.cachesReceivedFromJoin(nodeId); - if (started != null) { - for (DynamicCacheDescriptor desc : started) { - IgnitePredicate filter = desc.groupDescriptor().config().getNodeFilter(); - - if (CU.affinityNode(ctx.discovery().localNode(), filter)) { - prepareCacheStart( - desc.cacheConfiguration(), - desc, - null, - exchTopVer - ); - } + for (DynamicCacheDescriptor desc : started) { + IgnitePredicate filter = desc.groupDescriptor().config().getNodeFilter(); + + if (CU.affinityNode(ctx.discovery().localNode(), filter)) { + prepareCacheStart( + desc.cacheConfiguration(), + desc, + null, + exchTopVer + ); } } - return started != null ? started : Collections.emptyList(); + return started; } /** From a790dface730eae11f98c6b6f74232f0ec32cf6b Mon Sep 17 00:00:00 2001 From: Ivan Rakov Date: Wed, 26 Jul 2017 13:41:33 +0300 Subject: [PATCH 029/547] Failing flaky test with link to ticket number --- .../cache/persistence/db/IgnitePdsWholeClusterRestartTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsWholeClusterRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsWholeClusterRestartTest.java index c8ec304e2a5cf..1cdfaea659465 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsWholeClusterRestartTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsWholeClusterRestartTest.java @@ -118,6 +118,8 @@ private void deleteWorkFiles() throws IgniteCheckedException { * @throws Exception if failed. */ public void testRestarts() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-5741"); + startGrids(GRID_CNT); ignite(0).active(true); From e001a063a9c2260c4732093e54c332cb8af33b0b Mon Sep 17 00:00:00 2001 From: EdShangGG Date: Wed, 26 Jul 2017 16:37:45 +0300 Subject: [PATCH 030/547] new utility methods for working with files --- .../ignite/internal/util/IgniteUtils.java | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java index 6a3be5521f70b..abdab698fd869 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java @@ -73,9 +73,13 @@ import java.nio.channels.SelectionKey; import java.nio.channels.Selector; import java.nio.charset.Charset; +import java.nio.file.DirectoryStream; +import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.security.AccessController; import java.security.KeyManagementException; import java.security.MessageDigest; @@ -9978,6 +9982,67 @@ public static byte[] zip(@Nullable byte[] bytes) throws IgniteCheckedException { } } + /** + * Return count of regular file in the directory (including in sub-directories) + * + * @param dir path to directory + * @return count of regular file + * @throws IOException sometimes + */ + public static int fileCount(Path dir) throws IOException { + int cnt = 0; + + try (DirectoryStream ds = Files.newDirectoryStream(dir)){ + for (Path d : ds) { + if (Files.isDirectory(d)) + cnt += fileCount(d); + + else if (Files.isRegularFile(d)) + cnt++; + } + } + + return cnt; + } + + /** + * Will calculate the size of a directory. + * + * If there is concurrent activity in the directory, than returned value may be wrong. + */ + public static long dirSize(Path path) throws IgniteCheckedException { + final AtomicLong s = new AtomicLong(0); + + try { + Files.walkFileTree(path, new SimpleFileVisitor() { + @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + s.addAndGet(attrs.size()); + + return FileVisitResult.CONTINUE; + } + + @Override public FileVisitResult visitFileFailed(Path file, IOException exc) { + U.error(null, "file skipped - " + file, exc); + + // Skip directory or file + return FileVisitResult.CONTINUE; + } + + @Override public FileVisitResult postVisitDirectory(Path dir, IOException exc) { + if (exc != null) + U.error(null, "error during size calculation of directory - " + dir, exc); + + // Ignoring + return FileVisitResult.CONTINUE; + } + }); + } catch (IOException e) { + throw new IgniteCheckedException("walkFileTree will not throw IOException if the FileVisitor does not"); + } + + return s.get(); + } + /** * Returns {@link GridIntIterator} for range of primitive integers. * @param start Start. From 50c5b1dfc1cf182f42bd36dc5d5757f61180ff36 Mon Sep 17 00:00:00 2001 From: EdShangGG Date: Wed, 26 Jul 2017 16:38:08 +0300 Subject: [PATCH 031/547] small improvements in abstract tests -checking "checkTopology" flag in startGridsMultiThreaded -auto cluster activation in startGridsMultiThreaded --- .../apache/ignite/testframework/junits/GridAbstractTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java index d6d241ccbdac9..9b99e01b4585d 100755 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java @@ -698,7 +698,9 @@ protected Ignite startGridsMultiThreaded(int cnt) throws Exception { if (cnt > 1) { startGridsMultiThreaded(1, cnt - 1); - checkTopology(cnt); + + if (checkTopology()) + checkTopology(cnt); } return ignite; From e96225f1ed4080243844108e798e3fa74d13d542 Mon Sep 17 00:00:00 2001 From: Dmitrii Ryabov Date: Wed, 26 Jul 2017 17:00:52 +0300 Subject: [PATCH 032/547] IGNITE-4648 IgniteInternalTx.prepare() does not wait for async operations to complete --- .../cache/GridCacheSharedContext.java | 6 +- .../distributed/near/GridNearTxLocal.java | 6 +- .../IgniteTxImplicitSingleStateImpl.java | 2 +- .../IgniteTxRemoteStateAdapter.java | 2 +- .../cache/transactions/IgniteTxState.java | 2 +- .../cache/transactions/IgniteTxStateImpl.java | 2 +- .../transactions/PlatformTransactions.java | 2 +- ...riginatingNodeFailureAbstractSelfTest.java | 2 +- ...GridCachePartitionedTxSalvageSelfTest.java | 8 +-- .../cache/jta/CacheJtaResource.java | 20 +++--- .../{ => jta}/AbstractCacheJtaSelfTest.java | 69 ++++++++++++++++++- ...ridPartitionedCacheJtaFactorySelfTest.java | 2 +- ...itionedCacheJtaFactoryUseSyncSelfTest.java | 2 +- ...tionedCacheJtaLookupClassNameSelfTest.java | 4 +- ...GridReplicatedCacheJtaFactorySelfTest.java | 2 +- ...licatedCacheJtaFactoryUseSyncSelfTest.java | 2 +- ...icatedCacheJtaLookupClassNameSelfTest.java | 2 +- .../processors/cache/jta/package-info.java | 22 ++++++ .../ignite/testsuites/IgniteJtaTestSuite.java | 12 ++-- .../Cache/CacheAbstractTransactionalTest.cs | 53 +++++++++----- 20 files changed, 165 insertions(+), 57 deletions(-) rename modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/{ => jta}/AbstractCacheJtaSelfTest.java (72%) rename modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/{ => jta}/GridPartitionedCacheJtaFactorySelfTest.java (96%) rename modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/{ => jta}/GridPartitionedCacheJtaFactoryUseSyncSelfTest.java (95%) rename modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/{ => jta}/GridPartitionedCacheJtaLookupClassNameSelfTest.java (96%) rename modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/{ => jta}/GridReplicatedCacheJtaFactorySelfTest.java (95%) rename modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/{ => jta}/GridReplicatedCacheJtaFactoryUseSyncSelfTest.java (95%) rename modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/{ => jta}/GridReplicatedCacheJtaLookupClassNameSelfTest.java (95%) create mode 100644 modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/package-info.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java index 5387cc8522790..1876023d48b47 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java @@ -910,7 +910,7 @@ public void cleanup() { * @throws IgniteCheckedException If failed. */ public void endTx(GridNearTxLocal tx) throws IgniteCheckedException { - tx.txState().awaitLastFut(this); + tx.txState().awaitLastFuture(this); tx.close(); } @@ -924,7 +924,7 @@ public IgniteInternalFuture commitTxAsync(GridNearTxLocal tx) GridCacheContext ctx = tx.txState().singleCacheContext(this); if (ctx == null) { - tx.txState().awaitLastFut(this); + tx.txState().awaitLastFuture(this); return tx.commitNearTxLocalAsync(); } @@ -938,7 +938,7 @@ public IgniteInternalFuture commitTxAsync(GridNearTxLocal tx) * @return Rollback future. */ public IgniteInternalFuture rollbackTxAsync(GridNearTxLocal tx) throws IgniteCheckedException { - tx.txState().awaitLastFut(this); + tx.txState().awaitLastFuture(this); return tx.rollbackNearTxLocalAsync(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java index 81e5ca86ab643..58ecee9cf6cf8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java @@ -3120,9 +3120,13 @@ public IgniteInternalFuture prepareNearTxLocal() { } /** + * @param awaitLastFuture If true - method will wait until transaction finish every action started before. * @throws IgniteCheckedException If failed. */ - public final void prepare() throws IgniteCheckedException { + public final void prepare(boolean awaitLastFuture) throws IgniteCheckedException { + if (awaitLastFuture) + txState().awaitLastFuture(cctx); + prepareAsync().get(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxImplicitSingleStateImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxImplicitSingleStateImpl.java index 7610d50845f14..886d0d6c18482 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxImplicitSingleStateImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxImplicitSingleStateImpl.java @@ -91,7 +91,7 @@ public class IgniteTxImplicitSingleStateImpl extends IgniteTxLocalStateAdapter { } /** {@inheritDoc} */ - @Override public void awaitLastFut(GridCacheSharedContext ctx) { + @Override public void awaitLastFuture(GridCacheSharedContext ctx) { if (cacheCtx == null) return; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxRemoteStateAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxRemoteStateAdapter.java index 86ae684617dbb..bcb900c85dd15 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxRemoteStateAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxRemoteStateAdapter.java @@ -44,7 +44,7 @@ public abstract class IgniteTxRemoteStateAdapter implements IgniteTxRemoteState } /** {@inheritDoc} */ - @Override public void awaitLastFut(GridCacheSharedContext cctx) { + @Override public void awaitLastFuture(GridCacheSharedContext cctx) { assert false; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxState.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxState.java index ee48ed7c2b5f5..1fe0d2af68028 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxState.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxState.java @@ -59,7 +59,7 @@ public interface IgniteTxState { /** * @param cctx Awaits for previous async operations on active caches to be completed. */ - public void awaitLastFut(GridCacheSharedContext cctx); + public void awaitLastFuture(GridCacheSharedContext cctx); /** * @param cctx Context. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxStateImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxStateImpl.java index ad4ca61011381..4f14b5ce1b79c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxStateImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxStateImpl.java @@ -107,7 +107,7 @@ public class IgniteTxStateImpl extends IgniteTxLocalStateAdapter { } /** {@inheritDoc} */ - @Override public void awaitLastFut(GridCacheSharedContext cctx) { + @Override public void awaitLastFuture(GridCacheSharedContext cctx) { for (int i = 0; i < activeCacheIds.size(); i++) { int cacheId = activeCacheIds.get(i); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/transactions/PlatformTransactions.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/transactions/PlatformTransactions.java index 8f343436b1b35..8baca9bf6861a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/transactions/PlatformTransactions.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/transactions/PlatformTransactions.java @@ -160,7 +160,7 @@ private Transaction tx(long id) { @Override public long processInLongOutLong(int type, long val) throws IgniteCheckedException { switch (type) { case OP_PREPARE: - ((TransactionProxyImpl)tx(val)).tx().prepare(); + ((TransactionProxyImpl)tx(val)).tx().prepare(true); return TRUE; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxPessimisticOriginatingNodeFailureAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxPessimisticOriginatingNodeFailureAbstractSelfTest.java index 0463c4610f613..751455582c4b9 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxPessimisticOriginatingNodeFailureAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxPessimisticOriginatingNodeFailureAbstractSelfTest.java @@ -349,7 +349,7 @@ private void checkPrimaryNodeCrash(final boolean commmit) throws Exception { assertTrue(txEx.pessimistic()); if (commmit) { - txEx.prepare(); + txEx.prepare(true); // Fail the node in the middle of transaction. info(">>> Stopping primary node " + primaryNode); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedTxSalvageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedTxSalvageSelfTest.java index 06fbe8fcdd83f..256e8da8726e9 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedTxSalvageSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedTxSalvageSelfTest.java @@ -152,7 +152,7 @@ public void testPessimisticTxSalvageAfterTimeout() throws Exception { * Check whether caches has no transactions after salvage timeout. * * @param mode Transaction mode (PESSIMISTIC, OPTIMISTIC). - * @param prepare Whether to prepare transaction state (i.e. call {@link GridNearTxLocal#prepare()}). + * @param prepare Whether to prepare transaction state (i.e. call {@link GridNearTxLocal#prepare(boolean)}). * @throws Exception If failed. */ private void checkSalvageAfterTimeout(TransactionConcurrency mode, boolean prepare) throws Exception { @@ -171,7 +171,7 @@ private void checkSalvageAfterTimeout(TransactionConcurrency mode, boolean prepa * * @param mode Transaction mode (PESSIMISTIC, OPTIMISTIC). * @param prepare Whether to prepare transaction state - * (i.e. call {@link GridNearTxLocal#prepare()}). + * (i.e. call {@link GridNearTxLocal#prepare(boolean)}). * @throws Exception If failed. */ private void checkSalvageBeforeTimeout(TransactionConcurrency mode, boolean prepare) throws Exception { @@ -197,7 +197,7 @@ private void checkSalvageBeforeTimeout(TransactionConcurrency mode, boolean prep * Start new transaction on the grid(0) and put some keys to it. * * @param mode Transaction mode (PESSIMISTIC, OPTIMISTIC). - * @param prepare Whether to prepare transaction state (i.e. call {@link GridNearTxLocal#prepare()}). + * @param prepare Whether to prepare transaction state (i.e. call {@link GridNearTxLocal#prepare(boolean)}). * @throws Exception If failed. */ private void startTxAndPutKeys(final TransactionConcurrency mode, final boolean prepare) throws Exception { @@ -216,7 +216,7 @@ private void startTxAndPutKeys(final TransactionConcurrency mode, final boolean c.put(key, "val" + key); if (prepare) - ((TransactionProxyImpl)tx).tx().prepare(); + ((TransactionProxyImpl)tx).tx().prepare(true); } catch (IgniteCheckedException e) { info("Failed to put keys to cache: " + e.getMessage()); diff --git a/modules/jta/src/main/java/org/apache/ignite/internal/processors/cache/jta/CacheJtaResource.java b/modules/jta/src/main/java/org/apache/ignite/internal/processors/cache/jta/CacheJtaResource.java index 649f7c41be7fd..c63dafaf58508 100644 --- a/modules/jta/src/main/java/org/apache/ignite/internal/processors/cache/jta/CacheJtaResource.java +++ b/modules/jta/src/main/java/org/apache/ignite/internal/processors/cache/jta/CacheJtaResource.java @@ -56,6 +56,9 @@ final class CacheJtaResource implements XAResource, Synchronization { /** */ private Xid xid; + /** */ + private final GridKernalContext ctx; + /** * @param cacheTx Cache jta. * @param ctx Kernal context. @@ -65,6 +68,7 @@ final class CacheJtaResource implements XAResource, Synchronization { assert ctx != null; this.cacheTx = cacheTx; + this.ctx = ctx; if (log == null) log = U.logger(ctx, logRef, CacheJtaResource.class); @@ -100,7 +104,7 @@ private void throwException(String msg, Throwable cause) throws XAException { log.debug("XA resource rollback(...) [xid=" + xid + "]"); try { - cacheTx.rollback(); + ctx.cache().context().rollbackTxAsync(cacheTx).get(); } catch (IgniteCheckedException e) { throwException("Failed to rollback cache transaction: " + e.getMessage(), e); @@ -118,7 +122,7 @@ private void throwException(String msg, Throwable cause) throws XAException { throw new XAException("Cache transaction is not in active state."); try { - cacheTx.prepare(); + cacheTx.prepare(true); } catch (IgniteCheckedException e) { throwException("Failed to prepare cache transaction.", e); @@ -146,7 +150,7 @@ private void throwException(String msg, Throwable cause) throws XAException { log.debug("XA resource commit(...) [xid=" + xid + ", onePhase=" + onePhase + "]"); try { - cacheTx.commit(); + ctx.cache().context().commitTxAsync(cacheTx).get(); } catch (IgniteCheckedException e) { throwException("Failed to commit cache transaction: " + e.getMessage(), e); @@ -161,9 +165,7 @@ private void throwException(String msg, Throwable cause) throws XAException { log.debug("XA resource forget(...) [xid=" + xid + "]"); try { - cacheTx.invalidate(true); - - cacheTx.commit(); + ctx.cache().context().rollbackTxAsync(cacheTx).get(); } catch (IgniteCheckedException e) { throwException("Failed to forget cache transaction: " + e.getMessage(), e); @@ -246,7 +248,7 @@ private StringBuilder addFlag(StringBuilder sb, int flags, int mask, String flag throw new CacheException("Cache transaction is not in active state."); try { - cacheTx.prepare(); + cacheTx.prepare(true); } catch (IgniteCheckedException e) { throw new CacheException("Failed to prepare cache transaction.", e); @@ -261,7 +263,7 @@ private StringBuilder addFlag(StringBuilder sb, int flags, int mask, String flag log.debug("Synchronization.afterCompletion(STATUS_COMMITTED) [xid=" + cacheTx.xid() + "]"); try { - cacheTx.commit(); + ctx.cache().context().commitTxAsync(cacheTx).get(); } catch (IgniteCheckedException e) { throw new CacheException("Failed to commit cache transaction.", e); @@ -274,7 +276,7 @@ private StringBuilder addFlag(StringBuilder sb, int flags, int mask, String flag log.debug("Synchronization.afterCompletion(STATUS_ROLLEDBACK) [xid=" + cacheTx.xid() + "]"); try { - cacheTx.rollback(); + ctx.cache().context().rollbackTxAsync(cacheTx).get(); } catch (IgniteCheckedException e) { throw new CacheException("Failed to rollback cache transaction.", e); diff --git a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/AbstractCacheJtaSelfTest.java b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/AbstractCacheJtaSelfTest.java similarity index 72% rename from modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/AbstractCacheJtaSelfTest.java rename to modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/AbstractCacheJtaSelfTest.java index 37af32c6a89f2..89fb72f2a98cf 100644 --- a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/AbstractCacheJtaSelfTest.java +++ b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/AbstractCacheJtaSelfTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.cache; +package org.apache.ignite.internal.processors.cache.jta; import javax.transaction.Status; import javax.transaction.UserTransaction; @@ -24,10 +24,16 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest; +import org.apache.ignite.testframework.GridTestSafeThreadFactory; import org.apache.ignite.transactions.Transaction; import org.objectweb.jotm.Jotm; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; import static org.apache.ignite.cache.CacheMode.PARTITIONED; +import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED; import static org.apache.ignite.transactions.TransactionState.ACTIVE; /** @@ -155,7 +161,7 @@ public void testJtaTwoCaches() throws Exception { cache2.put("key2", 2); assertEquals(0, (int)cache1.get("key")); - assertEquals(0, (int)cache1.get("key")); + assertEquals(0, (int)cache2.get("key")); assertEquals(1, (int)cache1.get("key1")); assertEquals(2, (int)cache2.get("key2")); @@ -180,4 +186,63 @@ public void testJtaTwoCaches() throws Exception { assertEquals(1, (int)cache1.get("key1")); assertEquals(2, (int)cache2.get("key2")); } + + /** + * @throws Exception If failed. + */ + public void testAsyncOpAwait() throws Exception { + final IgniteCache cache = jcache(); + + GridTestSafeThreadFactory factory = new GridTestSafeThreadFactory("JtaThread"); + + final CountDownLatch latch = new CountDownLatch(1); + + Callable c = new Callable() { + @Override public Object call() throws Exception { + assertNull(grid(0).transactions().tx()); + + UserTransaction jtaTx = jotm.getUserTransaction(); + + jtaTx.begin(); + + try { + cache.put("key1", 1); + + cache.putAsync("key", 1); + + assertEquals(grid(0).transactions().tx().state(), ACTIVE); + + latch.countDown(); + + info("Before JTA commit."); + } + finally { + jtaTx.commit(); + } + + info("After JTA commit."); + + assertEquals((Integer)1, cache.get("key")); + + return null; + } + }; + + Thread task = factory.newThread(c); + + try (Transaction tx = ignite(0).transactions().txStart(PESSIMISTIC, READ_COMMITTED)) { + cache.put("key", 0); + + task.start(); + + latch.await(); + + while (task.getState() != Thread.State.WAITING) + factory.checkError(); + + info("Before cache TX commit."); + + tx.commit(); + } + } } diff --git a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridPartitionedCacheJtaFactorySelfTest.java b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridPartitionedCacheJtaFactorySelfTest.java similarity index 96% rename from modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridPartitionedCacheJtaFactorySelfTest.java rename to modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridPartitionedCacheJtaFactorySelfTest.java index f07997453c144..f6fd5c7675540 100644 --- a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridPartitionedCacheJtaFactorySelfTest.java +++ b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridPartitionedCacheJtaFactorySelfTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.cache; +package org.apache.ignite.internal.processors.cache.jta; import javax.cache.configuration.Factory; import org.apache.ignite.configuration.IgniteConfiguration; diff --git a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridPartitionedCacheJtaFactoryUseSyncSelfTest.java b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridPartitionedCacheJtaFactoryUseSyncSelfTest.java similarity index 95% rename from modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridPartitionedCacheJtaFactoryUseSyncSelfTest.java rename to modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridPartitionedCacheJtaFactoryUseSyncSelfTest.java index 5e6deee251c37..b66452d3b91c5 100644 --- a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridPartitionedCacheJtaFactoryUseSyncSelfTest.java +++ b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridPartitionedCacheJtaFactoryUseSyncSelfTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.cache; +package org.apache.ignite.internal.processors.cache.jta; import org.apache.ignite.configuration.IgniteConfiguration; diff --git a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridPartitionedCacheJtaLookupClassNameSelfTest.java b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridPartitionedCacheJtaLookupClassNameSelfTest.java similarity index 96% rename from modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridPartitionedCacheJtaLookupClassNameSelfTest.java rename to modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridPartitionedCacheJtaLookupClassNameSelfTest.java index bb1e89c6d3ebe..7357f8ea9af80 100644 --- a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridPartitionedCacheJtaLookupClassNameSelfTest.java +++ b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridPartitionedCacheJtaLookupClassNameSelfTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.cache; +package org.apache.ignite.internal.processors.cache.jta; import java.util.concurrent.Callable; import javax.transaction.TransactionManager; @@ -41,7 +41,7 @@ public class GridPartitionedCacheJtaLookupClassNameSelfTest extends AbstractCach * */ @IgniteIgnore(value = "https://issues.apache.org/jira/browse/IGNITE-1094", forceFailure = true) - public void testUncompatibleTmLookup() { + public void testIncompatibleTmLookup() { final IgniteEx ignite = grid(0); final CacheConfiguration cacheCfg = new CacheConfiguration(DEFAULT_CACHE_NAME); diff --git a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridReplicatedCacheJtaFactorySelfTest.java b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridReplicatedCacheJtaFactorySelfTest.java similarity index 95% rename from modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridReplicatedCacheJtaFactorySelfTest.java rename to modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridReplicatedCacheJtaFactorySelfTest.java index 3885447b3eb4b..1efd99cf8e009 100644 --- a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridReplicatedCacheJtaFactorySelfTest.java +++ b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridReplicatedCacheJtaFactorySelfTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.cache; +package org.apache.ignite.internal.processors.cache.jta; import org.apache.ignite.cache.CacheMode; diff --git a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridReplicatedCacheJtaFactoryUseSyncSelfTest.java b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridReplicatedCacheJtaFactoryUseSyncSelfTest.java similarity index 95% rename from modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridReplicatedCacheJtaFactoryUseSyncSelfTest.java rename to modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridReplicatedCacheJtaFactoryUseSyncSelfTest.java index e25f5e89faa99..54610f95ebaa4 100644 --- a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridReplicatedCacheJtaFactoryUseSyncSelfTest.java +++ b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridReplicatedCacheJtaFactoryUseSyncSelfTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.cache; +package org.apache.ignite.internal.processors.cache.jta; import org.apache.ignite.configuration.IgniteConfiguration; diff --git a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridReplicatedCacheJtaLookupClassNameSelfTest.java b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridReplicatedCacheJtaLookupClassNameSelfTest.java similarity index 95% rename from modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridReplicatedCacheJtaLookupClassNameSelfTest.java rename to modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridReplicatedCacheJtaLookupClassNameSelfTest.java index 2b89ba1a28474..a1aa6cf318576 100644 --- a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridReplicatedCacheJtaLookupClassNameSelfTest.java +++ b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridReplicatedCacheJtaLookupClassNameSelfTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.cache; +package org.apache.ignite.internal.processors.cache.jta; import org.apache.ignite.cache.CacheMode; diff --git a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/package-info.java b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/package-info.java new file mode 100644 index 0000000000000..49b208d8b6736 --- /dev/null +++ b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + * Contains internal tests or test related classes and interfaces. + */ +package org.apache.ignite.internal.processors.cache.jta; \ No newline at end of file diff --git a/modules/jta/src/test/java/org/apache/ignite/testsuites/IgniteJtaTestSuite.java b/modules/jta/src/test/java/org/apache/ignite/testsuites/IgniteJtaTestSuite.java index 4ae5df06d54ec..677f48500df53 100644 --- a/modules/jta/src/test/java/org/apache/ignite/testsuites/IgniteJtaTestSuite.java +++ b/modules/jta/src/test/java/org/apache/ignite/testsuites/IgniteJtaTestSuite.java @@ -21,12 +21,12 @@ import org.apache.ignite.internal.processors.cache.CacheJndiTmFactorySelfTest; import org.apache.ignite.internal.processors.cache.GridCacheJtaConfigurationValidationSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheJtaFactoryConfigValidationSelfTest; -import org.apache.ignite.internal.processors.cache.GridPartitionedCacheJtaFactorySelfTest; -import org.apache.ignite.internal.processors.cache.GridPartitionedCacheJtaFactoryUseSyncSelfTest; -import org.apache.ignite.internal.processors.cache.GridPartitionedCacheJtaLookupClassNameSelfTest; -import org.apache.ignite.internal.processors.cache.GridReplicatedCacheJtaFactorySelfTest; -import org.apache.ignite.internal.processors.cache.GridReplicatedCacheJtaFactoryUseSyncSelfTest; -import org.apache.ignite.internal.processors.cache.GridReplicatedCacheJtaLookupClassNameSelfTest; +import org.apache.ignite.internal.processors.cache.jta.GridPartitionedCacheJtaFactorySelfTest; +import org.apache.ignite.internal.processors.cache.jta.GridPartitionedCacheJtaFactoryUseSyncSelfTest; +import org.apache.ignite.internal.processors.cache.jta.GridPartitionedCacheJtaLookupClassNameSelfTest; +import org.apache.ignite.internal.processors.cache.jta.GridReplicatedCacheJtaFactorySelfTest; +import org.apache.ignite.internal.processors.cache.jta.GridReplicatedCacheJtaFactoryUseSyncSelfTest; +import org.apache.ignite.internal.processors.cache.jta.GridReplicatedCacheJtaLookupClassNameSelfTest; import org.apache.ignite.internal.processors.cache.GridJtaLifecycleAwareSelfTest; import org.apache.ignite.testframework.IgniteTestSuite; diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTransactionalTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTransactionalTest.cs index 45d0b5f10a74f..77ae8feb49ce4 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTransactionalTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTransactionalTest.cs @@ -606,8 +606,7 @@ public void TestTransactionScopeSingleCache() /// with multiple participating caches. /// [Test] - [Ignore("IGNITE-1561")] - public void TestTransactionScopeMultiCache() + public void TestTransactionScopeMultiCache([Values(true, false)] bool async) { var cache1 = Cache(); @@ -622,8 +621,16 @@ public void TestTransactionScopeMultiCache() // Commit. using (var ts = new TransactionScope()) { - cache1[1] = 10; - cache2[1] = 20; + if (async) + { + cache1.PutAsync(1, 10); + cache2.PutAsync(1, 20); + } + else + { + cache1.Put(1, 10); + cache2.Put(1, 20); + } ts.Complete(); } @@ -634,8 +641,16 @@ public void TestTransactionScopeMultiCache() // Rollback. using (new TransactionScope()) { - cache1[1] = 100; - cache2[1] = 200; + if (async) + { + cache1.PutAsync(1, 100); + cache2.PutAsync(1, 200); + } + else + { + cache1.Put(1, 100); + cache2.Put(1, 200); + } } Assert.AreEqual(10, cache1[1]); @@ -773,10 +788,10 @@ public void TestTransactionScopeAllOperations() for (var i = 0; i < 10; i++) { CheckTxOp((cache, key) => cache.Put(key, -5)); - CheckTxOp((cache, key) => cache.PutAsync(key, -5).Wait()); + CheckTxOp((cache, key) => cache.PutAsync(key, -5)); CheckTxOp((cache, key) => cache.PutAll(new Dictionary {{key, -7}})); - CheckTxOp((cache, key) => cache.PutAllAsync(new Dictionary {{key, -7}}).Wait()); + CheckTxOp((cache, key) => cache.PutAllAsync(new Dictionary {{key, -7}})); CheckTxOp((cache, key) => { @@ -786,11 +801,11 @@ public void TestTransactionScopeAllOperations() CheckTxOp((cache, key) => { cache.Remove(key); - cache.PutIfAbsentAsync(key, -10).Wait(); + cache.PutIfAbsentAsync(key, -10); }); CheckTxOp((cache, key) => cache.GetAndPut(key, -9)); - CheckTxOp((cache, key) => cache.GetAndPutAsync(key, -9).Wait()); + CheckTxOp((cache, key) => cache.GetAndPutAsync(key, -9)); CheckTxOp((cache, key) => { @@ -800,32 +815,32 @@ public void TestTransactionScopeAllOperations() CheckTxOp((cache, key) => { cache.Remove(key); - cache.GetAndPutIfAbsentAsync(key, -10).Wait(); + cache.GetAndPutIfAbsentAsync(key, -10); }); CheckTxOp((cache, key) => cache.GetAndRemove(key)); - CheckTxOp((cache, key) => cache.GetAndRemoveAsync(key).Wait()); + CheckTxOp((cache, key) => cache.GetAndRemoveAsync(key)); CheckTxOp((cache, key) => cache.GetAndReplace(key, -11)); - CheckTxOp((cache, key) => cache.GetAndReplaceAsync(key, -11).Wait()); + CheckTxOp((cache, key) => cache.GetAndReplaceAsync(key, -11)); CheckTxOp((cache, key) => cache.Invoke(key, new AddProcessor(), 1)); - CheckTxOp((cache, key) => cache.InvokeAsync(key, new AddProcessor(), 1).Wait()); + CheckTxOp((cache, key) => cache.InvokeAsync(key, new AddProcessor(), 1)); CheckTxOp((cache, key) => cache.InvokeAll(new[] {key}, new AddProcessor(), 1)); - CheckTxOp((cache, key) => cache.InvokeAllAsync(new[] {key}, new AddProcessor(), 1).Wait()); + CheckTxOp((cache, key) => cache.InvokeAllAsync(new[] {key}, new AddProcessor(), 1)); CheckTxOp((cache, key) => cache.Remove(key)); - CheckTxOp((cache, key) => cache.RemoveAsync(key).Wait()); + CheckTxOp((cache, key) => cache.RemoveAsync(key)); CheckTxOp((cache, key) => cache.RemoveAll(new[] {key})); - CheckTxOp((cache, key) => cache.RemoveAllAsync(new[] {key}).Wait()); + CheckTxOp((cache, key) => cache.RemoveAllAsync(new[] {key})); CheckTxOp((cache, key) => cache.Replace(key, 100)); - CheckTxOp((cache, key) => cache.ReplaceAsync(key, 100).Wait()); + CheckTxOp((cache, key) => cache.ReplaceAsync(key, 100)); CheckTxOp((cache, key) => cache.Replace(key, cache[key], 100)); - CheckTxOp((cache, key) => cache.ReplaceAsync(key, cache[key], 100).Wait()); + CheckTxOp((cache, key) => cache.ReplaceAsync(key, cache[key], 100)); } } From 9e79c4b69da2e553d845edcfa5d61717a8ebb5f0 Mon Sep 17 00:00:00 2001 From: dpavlov Date: Wed, 26 Jul 2017 17:23:05 +0300 Subject: [PATCH 033/547] IGNITE-5806 - Fixed assertion with a side-effect - Fixes #2335. Signed-off-by: Alexey Goncharuk --- .../processors/cache/store/GridCacheStoreManagerAdapter.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java index 99541ba81a7db..9cba3ddadb8c4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java @@ -875,9 +875,8 @@ private void sessionEnd0(@Nullable IgniteInternalTx tx, boolean threwEx) throws lsnr.onSessionEnd(locSes, !threwEx); } - assert !sesHolder.get().ended(store); - - store.sessionEnd(!threwEx); + if (!sesHolder.get().ended(store)) + store.sessionEnd(!threwEx); } } catch (Exception e) { From 586a96eaf4570f4f2020041cfef07550025421d8 Mon Sep 17 00:00:00 2001 From: Evgeny Stanilovskiy Date: Wed, 26 Jul 2017 18:49:41 +0300 Subject: [PATCH 034/547] IGNITE-5174: list only server nodes for specified topology version Fixes #2312 --- .../managers/discovery/GridDiscoveryManager.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java index 9f5bd3f02272e..7ad058db71213 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java @@ -1974,6 +1974,16 @@ private DiscoCache resolveDiscoCache(int grpId, AffinityTopologyVersion topVer) return snapshots.get(topVer); } + /** + * Gets server nodes topology by specified version from snapshots history storage. + * + * @param topVer Topology version. + * @return Server topology nodes or {@code null} if there are no nodes for passed in version. + */ + @Nullable public Collection serverTopologyNodes(long topVer) { + return F.view(topology(topVer), F.not(FILTER_CLI)); + } + /** @return All daemon nodes in topology. */ public Collection daemonNodes() { return discoCache().daemonNodes(); From b49469f469e3ad9f64d009a474b6e09686ec9203 Mon Sep 17 00:00:00 2001 From: Dmitriy Shabalin Date: Thu, 27 Jul 2017 10:39:54 +0700 Subject: [PATCH 035/547] IGNITE-5835 Web Console: Highlight active element in select input. --- .../web-console/frontend/app/app.config.js | 3 +- .../app/components/bs-select-menu/style.scss | 16 +++++ .../components/bs-select-menu/template.pug | 2 + .../app/primitives/dropdown/index.scss | 68 ------------------- 4 files changed, 20 insertions(+), 69 deletions(-) diff --git a/modules/web-console/frontend/app/app.config.js b/modules/web-console/frontend/app/app.config.js index b9741d124fe02..6ba1d9865c920 100644 --- a/modules/web-console/frontend/app/app.config.js +++ b/modules/web-console/frontend/app/app.config.js @@ -78,7 +78,8 @@ igniteConsoleCfg.config(['$selectProvider', ($selectProvider) => { noneText: 'Clear All', template: '', iconCheckmark: 'fa fa-check', - caretHtml: '' + caretHtml: '', + animation: '' }); }]); diff --git a/modules/web-console/frontend/app/components/bs-select-menu/style.scss b/modules/web-console/frontend/app/components/bs-select-menu/style.scss index faddd3ce4848c..870b1bf19d7ed 100644 --- a/modules/web-console/frontend/app/components/bs-select-menu/style.scss +++ b/modules/web-console/frontend/app/components/bs-select-menu/style.scss @@ -75,4 +75,20 @@ padding-bottom: 10px; } } + + &.bssm-multiple { + .bssm-active-indicator { + display: initial; + } + } + + &:not(.bssm-multiple) { + .bssm-active-indicator { + display: none; + } + + & > li > .bssm-item-button__active { + background-color: #eeeeee; + } + } } \ No newline at end of file diff --git a/modules/web-console/frontend/app/components/bs-select-menu/template.pug b/modules/web-console/frontend/app/components/bs-select-menu/template.pug index 91f3d8ef7ffe8..a9c1c2881727a 100644 --- a/modules/web-console/frontend/app/components/bs-select-menu/template.pug +++ b/modules/web-console/frontend/app/components/bs-select-menu/template.pug @@ -17,6 +17,7 @@ ul.bs-select-menu( tabindex='-1' ng-show='$isVisible()' + ng-class=`{ 'bssm-multiple': $isMultiple }` role='select' ) li(ng-if='$showAllNoneButtons || ($isMultiple && $matches.length > 2)') @@ -37,6 +38,7 @@ ul.bs-select-menu( role='menuitem' tabindex='-1' ng-click='$select($index, $event)' + ng-class=`{ 'bssm-item-button__active': $isActive($index) }` data-placement='right auto' title='{{ ::match.label }}' ) diff --git a/modules/web-console/frontend/app/primitives/dropdown/index.scss b/modules/web-console/frontend/app/primitives/dropdown/index.scss index 47a7e900aa648..4695d211dab2a 100644 --- a/modules/web-console/frontend/app/primitives/dropdown/index.scss +++ b/modules/web-console/frontend/app/primitives/dropdown/index.scss @@ -53,74 +53,6 @@ border-bottom: 1px solid $table-border-color; } } - - ul.select.dropdown-menu { - li { - hr { - display: none; - } - - a { - &:before { - content: ''; - - position: relative; - top: 1px; - left: -3px; - - display: inline-block; - width: 12px; - height: 12px; - - margin-left: 0; - margin-right: 7px; - - border: 1px solid #afafaf; - border-radius: 2px; - background-color: #FFF; - - box-shadow: inset 0 1px 1px $input-border; - } - - &.active { - color: $text-color; - } - - &.active:before { - border-color: $brand-info; - background-color: $brand-info; - - box-shadow: none; - } - - & > i { - position: relative; - top: -2px; - left: -18px; - - float: none; - - width: 0; - margin: 0; - - background: none; - - &.fa-check:before { - content: ''; - - display: block; - width: 4px; - height: 8px; - - border: solid #FFF; - border-width: 0 2px 2px 0; - - transform: rotate(35deg); - } - } - } - } - } } .dropdown--ignite { From 995258f9a326bb5a08b1e004d92e2760c25f20c0 Mon Sep 17 00:00:00 2001 From: Vasiliy Sisko Date: Thu, 27 Jul 2017 11:06:52 +0700 Subject: [PATCH 036/547] IGNITE-5767 Web Console: Changed mapping for BINARY SQL type to byte[]. --- .../store/jdbc/CacheJdbcPojoStoreTest.java | 55 +++++++- .../cache/store/jdbc/model/BinaryTest.java | 122 ++++++++++++++++++ .../cache/store/jdbc/model/BinaryTestKey.java | 87 +++++++++++++ .../frontend/app/data/jdbc-types.json | 6 +- 4 files changed, 264 insertions(+), 6 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/BinaryTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/BinaryTestKey.java diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java index bb85cabbd25a5..ea2808fa7cd7c 100644 --- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java +++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java @@ -17,6 +17,7 @@ package org.apache.ignite.cache.store.jdbc; +import java.io.ByteArrayInputStream; import java.lang.reflect.Field; import java.sql.Connection; import java.sql.PreparedStatement; @@ -25,6 +26,7 @@ import java.sql.Timestamp; import java.sql.Types; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.UUID; import java.util.concurrent.ConcurrentLinkedQueue; @@ -33,6 +35,8 @@ import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.binary.BinaryObjectBuilder; import org.apache.ignite.cache.store.jdbc.dialect.H2Dialect; +import org.apache.ignite.cache.store.jdbc.model.BinaryTest; +import org.apache.ignite.cache.store.jdbc.model.BinaryTestKey; import org.apache.ignite.cache.store.jdbc.model.Organization; import org.apache.ignite.cache.store.jdbc.model.OrganizationKey; import org.apache.ignite.cache.store.jdbc.model.Person; @@ -44,7 +48,6 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiInClosure; import org.apache.ignite.testframework.GridTestUtils; -import org.apache.ignite.testframework.config.GridTestProperties; import org.apache.ignite.testframework.junits.cache.GridAbstractCacheStoreSelfTest; import org.h2.jdbcx.JdbcConnectionPool; @@ -78,7 +81,7 @@ public CacheJdbcPojoStoreTest() throws Exception { @Override protected CacheJdbcPojoStore store() { CacheJdbcPojoStoreFactory storeFactory = new CacheJdbcPojoStoreFactory<>(); - JdbcType[] storeTypes = new JdbcType[6]; + JdbcType[] storeTypes = new JdbcType[7]; storeTypes[0] = new JdbcType(); storeTypes[0].setDatabaseSchema("PUBLIC"); @@ -147,6 +150,15 @@ public CacheJdbcPojoStoreTest() throws Exception { storeTypes[5].setValueType("java.util.UUID"); storeTypes[5].setValueFields(new JdbcTypeField(Types.BINARY, "VAL", UUID.class, null)); + storeTypes[6] = new JdbcType(); + storeTypes[6].setDatabaseSchema("PUBLIC"); + storeTypes[6].setDatabaseTable("BINARY_ENTRIES"); + storeTypes[6].setKeyType("org.apache.ignite.cache.store.jdbc.model.BinaryTestKey"); + storeTypes[6].setKeyFields(new JdbcTypeField(Types.BINARY, "KEY", Integer.class, "id")); + + storeTypes[6].setValueType("org.apache.ignite.cache.store.jdbc.model.BinaryTest"); + storeTypes[6].setValueFields(new JdbcTypeField(Types.BINARY, "VAL", byte[].class, "bytes")); + storeFactory.setTypes(storeTypes); storeFactory.setDialect(new H2Dialect()); @@ -210,12 +222,22 @@ public CacheJdbcPojoStoreTest() throws Exception { // No-op. } + try { + stmt.executeUpdate("delete from Binary_Entries"); + } + catch (SQLException ignore) { + // No-op. + } + stmt.executeUpdate("CREATE TABLE IF NOT EXISTS " + "String_Entries (key varchar(100) not null, val varchar(100), PRIMARY KEY(key))"); stmt.executeUpdate("CREATE TABLE IF NOT EXISTS " + "UUID_Entries (key binary(16) not null, val binary(16), PRIMARY KEY(key))"); + stmt.executeUpdate("CREATE TABLE IF NOT EXISTS " + + "Binary_Entries (key binary(16) not null, val binary(16), PRIMARY KEY(key))"); + stmt.executeUpdate("CREATE TABLE IF NOT EXISTS " + "Timestamp_Entries (key timestamp not null, val integer, PRIMARY KEY(key))"); @@ -304,11 +326,31 @@ public void testLoadCache() throws Exception { conn.commit(); + U.closeQuiet(prnStmt); + + PreparedStatement binaryStmt = conn.prepareStatement("INSERT INTO Binary_Entries(key, val) VALUES (?, ?)"); + + byte[] bytes = new byte[16]; + + for (byte i = 0; i < 16; i++) + bytes[i] = i; + + binaryStmt.setInt(1, 1); + binaryStmt.setBinaryStream(2, new ByteArrayInputStream(bytes)); + + binaryStmt.addBatch(); + binaryStmt.executeBatch(); + + U.closeQuiet(binaryStmt); + + conn.commit(); + U.closeQuiet(conn); final Collection orgKeys = new ConcurrentLinkedQueue<>(); final Collection prnKeys = new ConcurrentLinkedQueue<>(); final Collection prnComplexKeys = new ConcurrentLinkedQueue<>(); + final Collection binaryTestVals = new ConcurrentLinkedQueue<>(); IgniteBiInClosure c = new CI2() { @Override public void apply(Object k, Object v) { @@ -331,12 +373,18 @@ public void testLoadCache() throws Exception { if (PersonComplexKey.class.getName().equals(keyType) && Person.class.getName().equals(valType)) prnComplexKeys.add(key); + + if (BinaryTestKey.class.getName().equals(keyType) + && BinaryTest.class.getName().equals(valType)) + binaryTestVals.add(val.field("bytes")); } }else { if (k instanceof OrganizationKey && v instanceof Organization) orgKeys.add(k); else if (k instanceof PersonKey && v instanceof Person) prnKeys.add(k); + else if (k instanceof BinaryTestKey && v instanceof BinaryTest) + binaryTestVals.add(((BinaryTest)v).getBytes()); else if (k instanceof PersonComplexKey && v instanceof Person) { PersonComplexKey key = (PersonComplexKey)k; @@ -357,6 +405,8 @@ else if (k instanceof PersonComplexKey && v instanceof Person) { assertEquals(ORGANIZATION_CNT, orgKeys.size()); assertEquals(PERSON_CNT, prnKeys.size()); assertEquals(PERSON_CNT, prnComplexKeys.size()); + assertEquals(1, binaryTestVals.size()); + assertTrue(Arrays.equals(bytes, (byte[])binaryTestVals.iterator().next())); Collection tmpOrgKeys = new ArrayList<>(orgKeys); Collection tmpPrnKeys = new ArrayList<>(prnKeys); @@ -545,5 +595,4 @@ private Object wrap(Object obj) throws IllegalAccessException { return obj; } - } diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/BinaryTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/BinaryTest.java new file mode 100644 index 0000000000000..6cb551b10dd6b --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/BinaryTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.cache.store.jdbc.model; + +import java.io.Serializable; +import java.sql.Date; +import java.util.Arrays; + +/** + * BinaryTest definition. + */ +public class BinaryTest implements Serializable { + /** */ + private static final long serialVersionUID = 0L; + + /** Value for id. */ + private Integer id; + + /** Value for bytes. */ + private byte[] bytes; + + /** + * Empty constructor. + */ + public BinaryTest() { + // No-op. + } + + /** + * Full constructor. + */ + public BinaryTest( + Integer id, + byte[] bytes + ) { + this.id = id; + this.bytes = bytes; + } + + /** + * Gets id. + * + * @return Value for id. + */ + public Integer getId() { + return id; + } + + /** + * Sets id. + * + * @param id New value for id. + */ + public void setId(Integer id) { + this.id = id; + } + + /** + * Gets bytes. + * + * @return Value for bytes. + */ + public byte[] getBytes() { + return bytes; + } + + /** + * Sets bytes. + * + * @param bytes New value for bytes. + */ + public void setBytes(byte[] bytes) { + this.bytes = bytes; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + + if (!(o instanceof BinaryTest)) + return false; + + BinaryTest that = (BinaryTest)o; + + if (id != null ? !id.equals(that.id) : that.id != null) + return false; + + return bytes != null ? Arrays.equals(bytes, that.bytes) : that.bytes == null; + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + int res = id != null ? id.hashCode() : 0; + + res = 31 * res + (bytes != null ? Arrays.hashCode(bytes) : 0); + + return res; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return "Person [id=" + id + + ", bytes=" + Arrays.toString(bytes) + + "]"; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/BinaryTestKey.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/BinaryTestKey.java new file mode 100644 index 0000000000000..b710fad16af8c --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/model/BinaryTestKey.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.cache.store.jdbc.model; + +import java.io.Serializable; + +/** + * BinaryTestKey definition. + */ +public class BinaryTestKey implements Serializable { + /** */ + private static final long serialVersionUID = 0L; + + /** Value for id. */ + private Integer id; + + /** + * Empty constructor. + */ + public BinaryTestKey() { + // No-op. + } + + /** + * Full constructor. + */ + public BinaryTestKey(Integer id) { + this.id = id; + } + + /** + * Gets id. + * + * @return Value for id. + */ + public Integer getId() { + return id; + } + + /** + * Sets id. + * + * @param id New value for id. + */ + public void setId(Integer id) { + this.id = id; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + + if (!(o instanceof BinaryTestKey)) + return false; + + BinaryTestKey that = (BinaryTestKey)o; + + return id != null ? id.equals(that.id) : that.id == null; + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return id != null ? id.hashCode() : 0; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return "PersonKey [id=" + id + + "]"; + } +} diff --git a/modules/web-console/frontend/app/data/jdbc-types.json b/modules/web-console/frontend/app/data/jdbc-types.json index 07abbaf998016..c743e315bfd35 100644 --- a/modules/web-console/frontend/app/data/jdbc-types.json +++ b/modules/web-console/frontend/app/data/jdbc-types.json @@ -21,9 +21,9 @@ {"dbName": "DATE", "dbType": 91, "signed": {"javaType": "Date"}}, {"dbName": "TIME", "dbType": 92, "signed": {"javaType": "Time"}}, {"dbName": "TIMESTAMP", "dbType": 93, "signed": {"javaType": "Timestamp"}}, - {"dbName": "BINARY", "dbType": -2, "signed": {"javaType": "Object"}}, - {"dbName": "VARBINARY", "dbType": -3, "signed": {"javaType": "Object"}}, - {"dbName": "LONGVARBINARY", "dbType": -4, "signed": {"javaType": "Object"}}, + {"dbName": "BINARY", "dbType": -2, "signed": {"javaType": "byte[]"}}, + {"dbName": "VARBINARY", "dbType": -3, "signed": {"javaType": "byte[]"}}, + {"dbName": "LONGVARBINARY", "dbType": -4, "signed": {"javaType": "byte[]"}}, {"dbName": "NULL", "dbType": 0, "signed": {"javaType": "Object"}}, {"dbName": "OTHER", "dbType": 1111, "signed": {"javaType": "Object"}}, {"dbName": "JAVA_OBJECT", "dbType": 2000, "signed": {"javaType": "Object"}}, From bcbb10d4ed72c3ac2cd8149bb5cd148f63e95725 Mon Sep 17 00:00:00 2001 From: Igor Seliverstov Date: Thu, 27 Jul 2017 09:44:34 +0300 Subject: [PATCH 037/547] IGNITE-5761 Add correct message when entries are not mapped to at least one node and avoid hang on rollback --- ...OptimisticSerializableTxPrepareFuture.java | 9 + .../GridNearOptimisticTxPrepareFuture.java | 15 ++ .../GridNearPessimisticTxPrepareFuture.java | 9 +- .../dht/NotMappedPartitionInTxTest.java | 247 ++++++++++++++++++ .../testsuites/IgniteCacheTestSuite5.java | 2 + 5 files changed, 281 insertions(+), 1 deletion(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/NotMappedPartitionInTxTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java index 72ddc67e0a291..561c4f7226d3b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java @@ -29,6 +29,7 @@ import org.apache.ignite.cluster.ClusterTopologyException; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; +import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheEntryEx; @@ -611,6 +612,14 @@ private void map( cacheCtx.affinity().nodesByKey(entry.key(), topVer) : cacheCtx.topology().nodes(cacheCtx.affinity().partition(entry.key()), topVer); + if (F.isEmpty(nodes)) { + onDone(new ClusterTopologyServerNotFoundException("Failed to map keys to nodes " + + "(partition is not mapped to any node) [key=" + entry.key() + + ", partition=" + cacheCtx.affinity().partition(entry.key()) + ", topVer=" + topVer + ']')); + + return; + } + txMapping.addMapping(nodes); ClusterNode primary = F.first(nodes); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java index edddf7dd78a6b..1e7a56770c124 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java @@ -32,6 +32,7 @@ import org.apache.ignite.cluster.ClusterTopologyException; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; +import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheEntryEx; @@ -430,6 +431,10 @@ private void prepare( GridDistributedTxMapping updated = map(write, topVer, cur, topLocked, remap); + if(updated == null) + // an exception occurred while transaction mapping, stop further processing + break; + if (write.context().isNear()) hasNearCache = true; @@ -640,6 +645,16 @@ private GridDistributedTxMapping map( cacheCtx.affinity().nodesByKey(entry.key(), topVer) : cacheCtx.topology().nodes(cacheCtx.affinity().partition(entry.key()), topVer); + if (F.isEmpty(nodes)) { + ClusterTopologyServerNotFoundException e = new ClusterTopologyServerNotFoundException("Failed to map " + + "keys to nodes (partition is not mapped to any node) [key=" + entry.key() + + ", partition=" + cacheCtx.affinity().partition(entry.key()) + ", topVer=" + topVer + ']'); + + onDone(e); + + return null; + } + txMapping.addMapping(nodes); ClusterNode primary = F.first(nodes); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java index e934319c91ebd..11cd9f93ab974 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java @@ -27,6 +27,7 @@ import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; +import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheEntryEx; @@ -282,7 +283,13 @@ private void preparePessimistic() { else nodes = cacheCtx.affinity().nodesByKey(txEntry.key(), topVer); - assert !nodes.isEmpty(); + if (F.isEmpty(nodes)) { + onDone(new ClusterTopologyServerNotFoundException("Failed to map keys to nodes (partition " + + "is not mapped to any node) [key=" + txEntry.key() + + ", partition=" + cacheCtx.affinity().partition(txEntry.key()) + ", topVer=" + topVer + ']')); + + return; + } ClusterNode primary = nodes.get(0); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/NotMappedPartitionInTxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/NotMappedPartitionInTxTest.java new file mode 100644 index 0000000000000..4059660ca5948 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/NotMappedPartitionInTxTest.java @@ -0,0 +1,247 @@ +package org.apache.ignite.internal.processors.cache.distributed.dht; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; +import java.util.concurrent.Callable; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.transactions.Transaction; +import org.apache.ignite.transactions.TransactionConcurrency; +import org.apache.ignite.transactions.TransactionIsolation; +import org.jetbrains.annotations.Nullable; + +import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC; +import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED; +import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; +import static org.apache.ignite.transactions.TransactionIsolation.SERIALIZABLE; + +/** + */ +@SuppressWarnings({"unchecked", "ThrowableNotThrown"}) +public class NotMappedPartitionInTxTest extends GridCommonAbstractTest { + /** Cache. */ + private static final String CACHE = "testCache"; + + /** Cache 2. */ + public static final String CACHE2 = CACHE + 1; + + /** Test key. */ + private static final String TEST_KEY = "key"; + + /** Is client. */ + private boolean isClient; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + return super.getConfiguration(gridName) + .setClientMode(isClient) + .setCacheConfiguration( + new CacheConfiguration(CACHE) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL) + .setCacheMode(CacheMode.REPLICATED) + .setAffinity(new TestAffinity()), + new CacheConfiguration(CACHE2) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)); + } + + /** + * + */ + public void testOneServerOptimistic() throws Exception { + try { + isClient = false; + startGrid(0); + + isClient = true; + final IgniteEx client = startGrid(1); + + GridTestUtils.assertThrowsAnyCause(log, new Callable() { + @Override public Void call() throws Exception { + testNotMapped(client, OPTIMISTIC, REPEATABLE_READ); + + return null; + } + }, ClusterTopologyServerNotFoundException.class, "Failed to map keys to nodes (partition is not mapped to any node)"); + } + finally { + stopAllGrids(); + } + } + + /** + * + */ + public void testOneServerOptimisticSerializable() throws Exception { + try { + isClient = false; + startGrid(0); + + isClient = true; + final IgniteEx client = startGrid(1); + + GridTestUtils.assertThrowsAnyCause(log, new Callable() { + @Override public Void call() throws Exception { + testNotMapped(client, OPTIMISTIC, SERIALIZABLE); + + return null; + } + }, ClusterTopologyServerNotFoundException.class, "Failed to map keys to nodes (partition is not mapped to any node)"); + } + finally { + stopAllGrids(); + } + } + + /** + * + */ + public void testOneServerPessimistic() throws Exception { + try { + isClient = false; + startGrid(0); + + isClient = true; + final IgniteEx client = startGrid(1); + + GridTestUtils.assertThrowsAnyCause(log, new Callable() { + @Override public Void call() throws Exception { + testNotMapped(client, PESSIMISTIC, READ_COMMITTED); + + return null; + } + }, ClusterTopologyServerNotFoundException.class, "Failed to lock keys (all partition nodes left the grid)"); + } + finally { + stopAllGrids(); + } + } + + /** + * + */ + public void testFourServersOptimistic() throws Exception { + try { + isClient = false; + startGrids(4); + + isClient = true; + final IgniteEx client = startGrid(4); + + GridTestUtils.assertThrowsAnyCause(log, new Callable() { + @Override public Void call() throws Exception { + testNotMapped(client, OPTIMISTIC, REPEATABLE_READ); + + return null; + } + }, ClusterTopologyServerNotFoundException.class, "Failed to map keys to nodes (partition is not mapped to any node)"); + } + finally { + stopAllGrids(); + } + } + + /** + * + */ + public void testFourServersOptimisticSerializable() throws Exception { + try { + isClient = false; + startGrids(4); + + isClient = true; + final IgniteEx client = startGrid(4); + + GridTestUtils.assertThrowsAnyCause(log, new Callable() { + @Override public Void call() throws Exception { + testNotMapped(client, OPTIMISTIC, SERIALIZABLE); + + return null; + } + }, ClusterTopologyServerNotFoundException.class, "Failed to map keys to nodes (partition is not mapped to any node)"); + } + finally { + stopAllGrids(); + } + } + + /** + * + */ + public void testFourServersPessimistic() throws Exception { + try { + isClient = false; + startGrids(4); + + isClient = true; + final IgniteEx client = startGrid(4); + + GridTestUtils.assertThrowsAnyCause(log, new Callable() { + @Override public Void call() throws Exception { + testNotMapped(client, PESSIMISTIC, READ_COMMITTED); + + return null; + } + }, ClusterTopologyServerNotFoundException.class, "Failed to lock keys (all partition nodes left the grid)"); + } + finally { + stopAllGrids(); + } + } + + /** + * @param client Ignite client. + */ + private void testNotMapped(IgniteEx client, TransactionConcurrency concurrency, TransactionIsolation isolation) { + IgniteCache cache2 = client.cache(CACHE2); + IgniteCache cache1 = client.cache(CACHE).withKeepBinary(); + + try(Transaction tx = client.transactions().txStart(concurrency, isolation)) { + + Map param = new TreeMap<>(); + param.put(TEST_KEY + 1, 1); + param.put(TEST_KEY + 1, 3); + param.put(TEST_KEY, 3); + + cache1.put(TEST_KEY, 3); + + cache1.putAll(param); + cache2.putAll(param); + + tx.commit(); + } + } + + /** */ + private static class TestAffinity extends RendezvousAffinityFunction { + /** {@inheritDoc} */ + @Override public int partition(Object key) { + if (TEST_KEY.equals(key)) + return 1; + + return super.partition(key); + } + + /** {@inheritDoc} */ + @Override public List assignPartition(int part, List nodes, int backups, + @Nullable Map> neighborhoodCache) { + if (part == 1) + return Collections.emptyList(); + + return super.assignPartition(part, nodes, backups, neighborhoodCache); + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java index 1395b954df600..dab2b19911a28 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java @@ -41,6 +41,7 @@ import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheGroupsPartitionLossPolicySelfTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteCachePartitionLossPolicySelfTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheTxIteratorSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.NotMappedPartitionInTxTest; import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.IgniteCacheAtomicProtocolTest; import org.apache.ignite.internal.processors.cache.distributed.rebalancing.CacheManualRebalancingTest; import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheSyncRebalanceModeSelfTest; @@ -94,6 +95,7 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(GridCachePartitionExchangeManagerHistSizeTest.class); suite.addTestSuite(GridCachePartitionEvictionDuringReadThroughSelfTest.class); + suite.addTestSuite(NotMappedPartitionInTxTest.class); return suite; } From 5172541f71e9878b4cc9df18580cdf1863a5820b Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Thu, 27 Jul 2017 10:41:41 +0300 Subject: [PATCH 038/547] IGNITE-5729 - IgniteCacheProxy instances from with() methods are not reusable after cache restart --- .../JettyRestProcessorAbstractSelfTest.java | 18 +- .../org/apache/ignite/cache/CacheManager.java | 4 +- .../cache/CacheAffinitySharedManager.java | 23 +- .../cache/CacheOperationContext.java | 15 + .../cache/GatewayProtectedCacheProxy.java | 1754 ++++++++++ .../processors/cache/GridCacheProcessor.java | 51 +- .../processors/cache/IgniteCacheProxy.java | 2818 +---------------- .../cache/IgniteCacheProxyImpl.java | 1810 +++++++++++ .../dr/IgniteDrDataStreamerCacheUpdater.java | 2 +- .../platform/cache/PlatformCache.java | 33 +- .../CacheEntryProcessorCopySelfTest.java | 11 +- .../cache/CacheStopAndDestroySelfTest.java | 1 + .../cache/GridCacheAbstractSelfTest.java | 2 + .../GridCacheOnCopyFlagAbstractSelfTest.java | 6 +- ...CacheValueConsistencyAbstractSelfTest.java | 4 +- ...gniteCacheConfigVariationsFullApiTest.java | 6 +- .../IgniteCacheEntryListenerAbstractTest.java | 1 - .../cache/IgniteCacheGroupsTest.java | 67 +- .../cache/IgniteCacheStartStopLoadTest.java | 1 - .../MemoryPolicyInitializationTest.java | 3 +- .../GridMarshallerAbstractTest.java | 10 +- 21 files changed, 3737 insertions(+), 2903 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GatewayProtectedCacheProxy.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java index 97321a713bab0..d99f2782d8c6b 100644 --- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java @@ -17,8 +17,6 @@ package org.apache.ignite.internal.processors.rest; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -38,6 +36,8 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.query.SqlQuery; @@ -76,8 +76,8 @@ import org.apache.ignite.internal.visor.cache.VisorCacheRebalanceTaskArg; import org.apache.ignite.internal.visor.cache.VisorCacheResetMetricsTask; import org.apache.ignite.internal.visor.cache.VisorCacheResetMetricsTaskArg; -import org.apache.ignite.internal.visor.cache.VisorCacheStartTaskArg; import org.apache.ignite.internal.visor.cache.VisorCacheStartTask; +import org.apache.ignite.internal.visor.cache.VisorCacheStartTaskArg; import org.apache.ignite.internal.visor.cache.VisorCacheStopTask; import org.apache.ignite.internal.visor.cache.VisorCacheStopTaskArg; import org.apache.ignite.internal.visor.compute.VisorComputeCancelSessionsTask; @@ -87,8 +87,8 @@ import org.apache.ignite.internal.visor.compute.VisorComputeToggleMonitoringTaskArg; import org.apache.ignite.internal.visor.compute.VisorGatewayTask; import org.apache.ignite.internal.visor.debug.VisorThreadDumpTask; -import org.apache.ignite.internal.visor.file.VisorFileBlockTaskArg; import org.apache.ignite.internal.visor.file.VisorFileBlockTask; +import org.apache.ignite.internal.visor.file.VisorFileBlockTaskArg; import org.apache.ignite.internal.visor.file.VisorLatestTextFilesTask; import org.apache.ignite.internal.visor.file.VisorLatestTextFilesTaskArg; import org.apache.ignite.internal.visor.igfs.VisorIgfsFormatTask; @@ -101,8 +101,8 @@ import org.apache.ignite.internal.visor.igfs.VisorIgfsResetMetricsTaskArg; import org.apache.ignite.internal.visor.igfs.VisorIgfsSamplingStateTask; import org.apache.ignite.internal.visor.igfs.VisorIgfsSamplingStateTaskArg; -import org.apache.ignite.internal.visor.log.VisorLogSearchTaskArg; import org.apache.ignite.internal.visor.log.VisorLogSearchTask; +import org.apache.ignite.internal.visor.log.VisorLogSearchTaskArg; import org.apache.ignite.internal.visor.misc.VisorAckTask; import org.apache.ignite.internal.visor.misc.VisorAckTaskArg; import org.apache.ignite.internal.visor.misc.VisorChangeGridActiveStateTask; @@ -121,16 +121,16 @@ import org.apache.ignite.internal.visor.node.VisorNodeSuppressedErrorsTaskArg; import org.apache.ignite.internal.visor.query.VisorQueryCancelTask; import org.apache.ignite.internal.visor.query.VisorQueryCancelTaskArg; +import org.apache.ignite.internal.visor.query.VisorQueryCleanupTask; import org.apache.ignite.internal.visor.query.VisorQueryCleanupTaskArg; import org.apache.ignite.internal.visor.query.VisorQueryDetailMetricsCollectorTask; import org.apache.ignite.internal.visor.query.VisorQueryDetailMetricsCollectorTaskArg; -import org.apache.ignite.internal.visor.query.VisorQueryResetMetricsTask; -import org.apache.ignite.internal.visor.query.VisorQueryResetMetricsTaskArg; -import org.apache.ignite.internal.visor.query.VisorQueryTaskArg; -import org.apache.ignite.internal.visor.query.VisorQueryCleanupTask; import org.apache.ignite.internal.visor.query.VisorQueryNextPageTask; import org.apache.ignite.internal.visor.query.VisorQueryNextPageTaskArg; +import org.apache.ignite.internal.visor.query.VisorQueryResetMetricsTask; +import org.apache.ignite.internal.visor.query.VisorQueryResetMetricsTaskArg; import org.apache.ignite.internal.visor.query.VisorQueryTask; +import org.apache.ignite.internal.visor.query.VisorQueryTaskArg; import org.apache.ignite.internal.visor.query.VisorRunningQueriesCollectorTask; import org.apache.ignite.internal.visor.query.VisorRunningQueriesCollectorTaskArg; import org.apache.ignite.lang.IgniteBiPredicate; diff --git a/modules/core/src/main/java/org/apache/ignite/cache/CacheManager.java b/modules/core/src/main/java/org/apache/ignite/cache/CacheManager.java index 351cd0da2e24c..03d1f40bcf550 100644 --- a/modules/core/src/main/java/org/apache/ignite/cache/CacheManager.java +++ b/modules/core/src/main/java/org/apache/ignite/cache/CacheManager.java @@ -43,7 +43,7 @@ import org.apache.ignite.internal.IgniteKernal; import org.apache.ignite.internal.IgnitionEx; import org.apache.ignite.internal.mxbean.IgniteStandardMXBean; -import org.apache.ignite.internal.processors.cache.IgniteCacheProxy; +import org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy; import org.apache.ignite.internal.util.IgniteUtils; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; @@ -177,7 +177,7 @@ public CacheManager(URI uri, CachingProvider cachingProvider, ClassLoader clsLdr if (res == null) throw new CacheException(); - ((IgniteCacheProxy)res).setCacheManager(this); + ((GatewayProtectedCacheProxy)res).setCacheManager(this); if (igniteCacheCfg.isManagementEnabled()) enableManagement(cacheName, true); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java index 79ab183485600..d251d52a9a7bf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java @@ -703,7 +703,7 @@ public void onCacheChangeRequest( assert cctx.cacheContext(cacheDesc.cacheId()) == null : "Starting cache has not null context: " + cacheDesc.cacheName(); - IgniteCacheProxy cacheProxy = cctx.cache().jcacheProxy(req.cacheName()); + IgniteCacheProxyImpl cacheProxy = (IgniteCacheProxyImpl) cctx.cache().jcacheProxy(req.cacheName()); // If it has proxy then try to start it if (cacheProxy != null) { @@ -772,28 +772,9 @@ public void onCacheChangeRequest( for (ExchangeActions.CacheActionData action : exchActions.cacheStopRequests()) cctx.cache().blockGateway(action.request().cacheName(), true, action.request().restart()); - for (ExchangeActions.CacheGroupActionData action : exchActions.cacheGroupsToStop()) { + for (ExchangeActions.CacheGroupActionData action : exchActions.cacheGroupsToStop()) cctx.exchange().clearClientTopology(action.descriptor().groupId()); - CacheGroupContext gctx = cctx.cache().cacheGroup(action.descriptor().groupId()); - - if (gctx != null) { - IgniteCheckedException ex; - - String msg = "Failed to wait for topology update, cache group is stopping."; - - // If snapshot operation in progress we must throw CacheStoppedException - // for correct cache proxy restart. For more details see - // IgniteCacheProxy.cacheException() - if (cctx.cache().context().snapshot().snapshotOperationInProgress()) - ex = new CacheStoppedException(msg); - else - ex = new IgniteCheckedException(msg); - - gctx.affinity().cancelFutures(ex); - } - } - Set stoppedGrps = null; if (crd) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheOperationContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheOperationContext.java index bcc0ee950ccac..eb2b9026cf7a9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheOperationContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheOperationContext.java @@ -232,6 +232,21 @@ public CacheOperationContext setNoRetries(boolean noRetries) { recovery); } + /** + * @param dataCenterId Data center id. + * @return Operation context. + */ + public CacheOperationContext setDataCenterId(byte dataCenterId) { + return new CacheOperationContext( + skipStore, + subjId, + keepBinary, + expiryPlc, + noRetries, + dataCenterId, + recovery); + } + /** * @param recovery Recovery flag. * @return New instance of CacheOperationContext with recovery flag. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GatewayProtectedCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GatewayProtectedCacheProxy.java new file mode 100644 index 0000000000000..5ba6810155440 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GatewayProtectedCacheProxy.java @@ -0,0 +1,1754 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.locks.Lock; +import javax.cache.CacheException; +import javax.cache.CacheManager; +import javax.cache.configuration.CacheEntryListenerConfiguration; +import javax.cache.configuration.Configuration; +import javax.cache.expiry.ExpiryPolicy; +import javax.cache.integration.CompletionListener; +import javax.cache.processor.EntryProcessor; +import javax.cache.processor.EntryProcessorResult; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheEntry; +import org.apache.ignite.cache.CacheEntryProcessor; +import org.apache.ignite.cache.CacheMetrics; +import org.apache.ignite.cache.CachePeekMode; +import org.apache.ignite.cache.query.FieldsQueryCursor; +import org.apache.ignite.cache.query.Query; +import org.apache.ignite.cache.query.QueryCursor; +import org.apache.ignite.cache.query.QueryDetailMetrics; +import org.apache.ignite.cache.query.QueryMetrics; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.cluster.ClusterGroup; +import org.apache.ignite.internal.AsyncSupportAdapter; +import org.apache.ignite.internal.util.tostring.GridToStringExclude; +import org.apache.ignite.lang.IgniteBiPredicate; +import org.apache.ignite.lang.IgniteClosure; +import org.apache.ignite.lang.IgniteFuture; +import org.apache.ignite.mxbean.CacheMetricsMXBean; +import org.apache.ignite.transactions.TransactionException; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * Cache proxy wrapper with gateway lock provided operations and possibility to change cache operation context. + */ +public class GatewayProtectedCacheProxy extends AsyncSupportAdapter> + implements IgniteCacheProxy { + /** */ + private static final long serialVersionUID = 0L; + + /** Cache proxy delegate. */ + private IgniteCacheProxy delegate; + + /** If {@code false} does not acquire read lock on gateway enter. */ + @GridToStringExclude private boolean lock; + + /** Cache operation context. */ + private CacheOperationContext opCtx; + + /** + * Empty constructor required for {@link Externalizable}. + */ + public GatewayProtectedCacheProxy() { + } + + /** + * + * @param delegate Cache proxy delegate. + * @param opCtx Cache operation context. + * @param lock True if cache proxy should be protected with gateway lock, false in other case. + */ + public GatewayProtectedCacheProxy( + @NotNull IgniteCacheProxy delegate, + @NotNull CacheOperationContext opCtx, + boolean lock + ) { + this.delegate = delegate; + this.opCtx = opCtx; + this.lock = lock; + } + + /** + * Sets CacheManager to delegate. + * + * @param cacheMgr Cache Manager. + */ + public void setCacheManager(org.apache.ignite.cache.CacheManager cacheMgr) { + if (delegate instanceof IgniteCacheProxyImpl) + ((IgniteCacheProxyImpl) delegate).setCacheManager(cacheMgr); + } + + /** {@inheritDoc} */ + @Override public GridCacheContext context() { + return delegate.context(); + } + + /** {@inheritDoc} */ + @Override public > C getConfiguration(Class clazz) { + return delegate.getConfiguration(clazz); + } + + /** {@inheritDoc} */ + @Override public String getName() { + return delegate.getName(); + } + + /** {@inheritDoc} */ + @Override public CacheManager getCacheManager() { + return delegate.getCacheManager(); + } + + /** {@inheritDoc} */ + @Override public GridCacheProxyImpl internalProxy() { + return delegate.internalProxy(); + } + + /** {@inheritDoc} */ + @Override public GatewayProtectedCacheProxy cacheNoGate() { + return new GatewayProtectedCacheProxy<>(delegate, opCtx, false); + } + + /** {@inheritDoc} */ + @Override public GatewayProtectedCacheProxy withExpiryPolicy(ExpiryPolicy plc) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return new GatewayProtectedCacheProxy<>(delegate, opCtx.withExpiryPolicy(plc), lock); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public GatewayProtectedCacheProxy withSkipStore() { + return skipStore(); + } + + /** {@inheritDoc} */ + @Override public GatewayProtectedCacheProxy skipStore() { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + boolean skip = opCtx.skipStore(); + + if (skip) + return this; + + return new GatewayProtectedCacheProxy<>(delegate, opCtx.setSkipStore(true), lock); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public GatewayProtectedCacheProxy withNoRetries() { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + boolean noRetries = opCtx.noRetries(); + + if (noRetries) + return this; + + return new GatewayProtectedCacheProxy<>(delegate, opCtx.setNoRetries(true), lock); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public GatewayProtectedCacheProxy withPartitionRecover() { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + boolean recovery = opCtx.recovery(); + + if (recovery) + return this; + + return new GatewayProtectedCacheProxy<>(delegate, opCtx.setRecovery(true), lock); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public GatewayProtectedCacheProxy withKeepBinary() { + return keepBinary(); + } + + /** {@inheritDoc} */ + @Override public GatewayProtectedCacheProxy keepBinary() { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return new GatewayProtectedCacheProxy<>((IgniteCacheProxy) delegate, opCtx.keepBinary(), lock); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public GatewayProtectedCacheProxy withDataCenterId(byte dataCenterId) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + Byte prevDataCenterId = opCtx.dataCenterId(); + + if (prevDataCenterId != null && dataCenterId == prevDataCenterId) + return this; + + return new GatewayProtectedCacheProxy<>(delegate, opCtx.setDataCenterId(dataCenterId), lock); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public void loadCache(@Nullable IgniteBiPredicate p, @Nullable Object... args) throws CacheException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + delegate.loadCache(p, args); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture loadCacheAsync(@Nullable IgniteBiPredicate p, @Nullable Object... args) throws CacheException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.loadCacheAsync(p, args); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public void localLoadCache(@Nullable IgniteBiPredicate p, @Nullable Object... args) throws CacheException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + delegate.localLoadCache(p, args); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture localLoadCacheAsync(@Nullable IgniteBiPredicate p, @Nullable Object... args) throws CacheException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.localLoadCacheAsync(p, args); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public V getAndPutIfAbsent(K key, V val) throws CacheException, TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.getAndPutIfAbsent(key, val); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture getAndPutIfAbsentAsync(K key, V val) throws CacheException, TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.getAndPutIfAbsentAsync(key, val); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public Lock lock(K key) { + return delegate.lock(key); + } + + /** {@inheritDoc} */ + @Override public Lock lockAll(Collection keys) { + return delegate.lockAll(keys); + } + + /** {@inheritDoc} */ + @Override public boolean isLocalLocked(K key, boolean byCurrThread) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.isLocalLocked(key, byCurrThread); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public QueryCursor query(Query qry) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.query(qry); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public FieldsQueryCursor> query(SqlFieldsQuery qry) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.query(qry); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public QueryCursor query(Query qry, IgniteClosure transformer) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.query(qry, transformer); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public Iterable> localEntries(CachePeekMode... peekModes) throws CacheException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.localEntries(peekModes); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public QueryMetrics queryMetrics() { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.queryMetrics(); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public void resetQueryMetrics() { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + delegate.resetQueryMetrics(); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public Collection queryDetailMetrics() { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.queryDetailMetrics(); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public void resetQueryDetailMetrics() { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + delegate.resetQueryDetailMetrics(); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public void localEvict(Collection keys) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + delegate.localEvict(keys); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public V localPeek(K key, CachePeekMode... peekModes) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.localPeek(key, peekModes); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public int size(CachePeekMode... peekModes) throws CacheException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.size(peekModes); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture sizeAsync(CachePeekMode... peekModes) throws CacheException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.sizeAsync(peekModes); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public long sizeLong(CachePeekMode... peekModes) throws CacheException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.sizeLong(peekModes); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture sizeLongAsync(CachePeekMode... peekModes) throws CacheException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.sizeLongAsync(peekModes); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public long sizeLong(int partition, CachePeekMode... peekModes) throws CacheException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.sizeLong(partition, peekModes); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture sizeLongAsync(int partition, CachePeekMode... peekModes) throws CacheException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.sizeLongAsync(partition, peekModes); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public int localSize(CachePeekMode... peekModes) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.localSize(peekModes); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public long localSizeLong(CachePeekMode... peekModes) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.localSizeLong(peekModes); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public long localSizeLong(int partition, CachePeekMode... peekModes) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.localSizeLong(partition, peekModes); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public Map> invokeAll(Map> map, Object... args) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.invokeAll(map, args); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture>> invokeAllAsync(Map> map, Object... args) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.invokeAllAsync(map, args); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public V get(K key) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.get(key); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture getAsync(K key) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.getAsync(key); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public CacheEntry getEntry(K key) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.getEntry(key); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture> getEntryAsync(K key) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.getEntryAsync(key); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public Map getAll(Set keys) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.getAll(keys); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture> getAllAsync(Set keys) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.getAllAsync(keys); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public Collection> getEntries(Set keys) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.getEntries(keys); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture>> getEntriesAsync(Set keys) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.getEntriesAsync(keys); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public Map getAllOutTx(Set keys) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.getAllOutTx(keys); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture> getAllOutTxAsync(Set keys) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.getAllOutTxAsync(keys); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public boolean containsKey(K key) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.containsKey(key); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public void loadAll(Set keys, boolean replaceExisting, CompletionListener completionListener) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + delegate.loadAll(keys, replaceExisting, completionListener); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture containsKeyAsync(K key) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.containsKeyAsync(key); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public boolean containsKeys(Set keys) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.containsKeys(keys); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture containsKeysAsync(Set keys) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.containsKeysAsync(keys); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public void put(K key, V val) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + delegate.put(key, val); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture putAsync(K key, V val) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.putAsync(key, val); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public V getAndPut(K key, V val) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.getAndPut(key, val); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture getAndPutAsync(K key, V val) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.getAndPutAsync(key, val); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public void putAll(Map map) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + delegate.putAll(map); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture putAllAsync(Map map) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.putAllAsync(map); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public boolean putIfAbsent(K key, V val) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.putIfAbsent(key, val); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture putIfAbsentAsync(K key, V val) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.putIfAbsentAsync(key, val); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public boolean remove(K key) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.remove(key); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture removeAsync(K key) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.removeAsync(key); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public boolean remove(K key, V oldVal) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.remove(key, oldVal); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture removeAsync(K key, V oldVal) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.removeAsync(key, oldVal); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public V getAndRemove(K key) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.getAndRemove(key); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture getAndRemoveAsync(K key) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.getAndRemoveAsync(key); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public boolean replace(K key, V oldVal, V newVal) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.replace(key, oldVal, newVal); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture replaceAsync(K key, V oldVal, V newVal) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.replaceAsync(key, oldVal, newVal); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public boolean replace(K key, V val) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.replace(key, val); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture replaceAsync(K key, V val) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.replaceAsync(key, val); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public V getAndReplace(K key, V val) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.getAndReplace(key, val); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture getAndReplaceAsync(K key, V val) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.getAndReplaceAsync(key, val); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public void removeAll(Set keys) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + delegate.removeAll(keys); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture removeAllAsync(Set keys) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.removeAllAsync(keys); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public void removeAll() { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + delegate.removeAll(); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture removeAllAsync() { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.removeAllAsync(); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public void clear() { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + delegate.clear(); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture clearAsync() { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.clearAsync(); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public void clear(K key) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + delegate.clear(key); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture clearAsync(K key) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.clearAsync(key); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public void clearAll(Set keys) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + delegate.clearAll(keys); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture clearAllAsync(Set keys) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.clearAllAsync(keys); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public void localClear(K key) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + delegate.localClear(key); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public void localClearAll(Set keys) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + delegate.localClearAll(keys); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public T invoke(K key, EntryProcessor entryProcessor, Object... arguments) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.invoke(key, entryProcessor, arguments); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture invokeAsync(K key, EntryProcessor entryProcessor, Object... arguments) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.invokeAsync(key, entryProcessor, arguments); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public T invoke(K key, CacheEntryProcessor entryProcessor, Object... arguments) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.invoke(key, entryProcessor, arguments); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture invokeAsync(K key, CacheEntryProcessor entryProcessor, Object... arguments) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.invokeAsync(key, entryProcessor, arguments); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public Map> invokeAll(Set keys, EntryProcessor entryProcessor, Object... args) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.invokeAll(keys, entryProcessor, args); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture>> invokeAllAsync(Set keys, EntryProcessor entryProcessor, Object... args) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.invokeAllAsync(keys, entryProcessor, args); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public Map> invokeAll(Set keys, CacheEntryProcessor entryProcessor, Object... args) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.invokeAll(keys, entryProcessor, args); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture>> invokeAllAsync(Set keys, CacheEntryProcessor entryProcessor, Object... args) throws TransactionException { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.invokeAllAsync(keys, entryProcessor, args); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public T unwrap(Class clazz) { + return delegate.unwrap(clazz); + } + + /** {@inheritDoc} */ + @Override public void registerCacheEntryListener(CacheEntryListenerConfiguration cacheEntryListenerConfiguration) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + delegate.registerCacheEntryListener(cacheEntryListenerConfiguration); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public void deregisterCacheEntryListener(CacheEntryListenerConfiguration cacheEntryListenerConfiguration) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + delegate.deregisterCacheEntryListener(cacheEntryListenerConfiguration); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public Iterator> iterator() { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.iterator(); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public void destroy() { + GridCacheGateway gate = gate(); + + if (!onEnterIfNoStop(gate)) + return; + + IgniteFuture destroyFuture; + + try { + destroyFuture = delegate.destroyAsync(); + } + finally { + onLeave(gate); + } + + if (destroyFuture != null) + destroyFuture.get(); + } + + /** {@inheritDoc} */ + @Override public IgniteFuture destroyAsync() { + return delegate.destroyAsync(); + } + + /** {@inheritDoc} */ + @Override public void close() { + GridCacheGateway gate = gate(); + + if (!onEnterIfNoStop(gate)) + return; + + IgniteFuture closeFuture; + + try { + closeFuture = closeAsync(); + } + finally { + onLeave(gate); + } + + if (closeFuture != null) + closeFuture.get(); + } + + /** {@inheritDoc} */ + @Override public IgniteFuture closeAsync() { + return delegate.closeAsync(); + } + + /** {@inheritDoc} */ + @Override public boolean isClosed() { + return delegate.isClosed(); + } + + /** {@inheritDoc} */ + @Override public IgniteFuture rebalance() { + return delegate.rebalance(); + } + + /** {@inheritDoc} */ + @Override public IgniteFuture indexReadyFuture() { + return delegate.indexReadyFuture(); + } + + /** {@inheritDoc} */ + @Override public CacheMetrics metrics() { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.metrics(); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public CacheMetrics metrics(ClusterGroup grp) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.metrics(grp); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public CacheMetrics localMetrics() { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.localMetrics(); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public CacheMetricsMXBean mxBean() { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.mxBean(); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public CacheMetricsMXBean localMxBean() { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.localMxBean(); + } + finally { + onLeave(gate, prev); + } + } + + /** {@inheritDoc} */ + @Override public Collection lostPartitions() { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.lostPartitions(); + } + finally { + onLeave(gate, prev); + } + } + + /** + * Safely get CacheGateway. + * + * @return Cache Gateway. + */ + @Nullable private GridCacheGateway gate() { + GridCacheContext cacheContext = delegate.context(); + return cacheContext != null ? cacheContext.gate() : null; + } + + /** + * Checks that proxy is in valid state (not closed, restarted or destroyed). + * Throws IllegalStateException or CacheRestartingException if proxy is in invalid state. + * + * @param gate Cache gateway. + */ + private void checkProxyIsValid(@Nullable GridCacheGateway gate) { + if (isProxyClosed()) + throw new IllegalStateException("Cache has been closed: " + context().name()); + + if (delegate instanceof IgniteCacheProxyImpl) + ((IgniteCacheProxyImpl) delegate).checkRestart(); + + if (gate == null) + throw new IllegalStateException("Gateway is unavailable. Probably cache has been destroyed, but proxy is not closed."); + } + + /** + * @param gate Cache gateway. + * @param opCtx Cache operation context to guard. + * @return Previous projection set on this thread. + */ + private CacheOperationContext onEnter(@Nullable GridCacheGateway gate, CacheOperationContext opCtx) { + checkProxyIsValid(gate); + + return lock ? gate.enter(opCtx) : gate.enterNoLock(opCtx); + } + + /** + * @param gate Cache gateway. + * @return {@code True} if enter successful. + */ + private boolean onEnterIfNoStop(@Nullable GridCacheGateway gate) { + try { + checkProxyIsValid(gate); + } + catch (Exception e) { + return false; + } + + return lock ? gate.enterIfNotStopped() : gate.enterIfNotStoppedNoLock(); + } + + /** + * @param gate Cache gateway. + * @param opCtx Operation context to guard. + */ + private void onLeave(GridCacheGateway gate, CacheOperationContext opCtx) { + if (lock) + gate.leave(opCtx); + else + gate.leaveNoLock(opCtx); + } + + /** + * @param gate Cache gateway. + */ + private void onLeave(GridCacheGateway gate) { + if (lock) + gate.leave(); + else + gate.leaveNoLock(); + } + + /** {@inheritDoc} */ + @Override public boolean isProxyClosed() { + return delegate.isProxyClosed(); + } + + /** {@inheritDoc} */ + @Override public void closeProxy() { + delegate.closeProxy(); + } + + /** {@inheritDoc} */ + @Override public V getTopologySafe(K key) { + return delegate.getTopologySafe(key); + } + + /** {@inheritDoc} */ + @Override public IgniteCache withAsync() { + return delegate.withAsync(); + } + + /** {@inheritDoc} */ + @Override public boolean isAsync() { + return delegate.isAsync(); + } + + /** {@inheritDoc} */ + @Override public IgniteFuture future() { + return delegate.future(); + } + + /** {@inheritDoc} */ + @Override public void writeExternal(ObjectOutput out) throws IOException { + out.writeObject(delegate); + + out.writeBoolean(lock); + + out.writeObject(opCtx); + } + + /** {@inheritDoc} */ + @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + delegate = (IgniteCacheProxy) in.readObject(); + + lock = in.readBoolean(); + + opCtx = (CacheOperationContext) in.readObject(); + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object another) { + GatewayProtectedCacheProxy anotherProxy = (GatewayProtectedCacheProxy) another; + + return delegate.equals(anotherProxy.delegate); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return delegate.hashCode(); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 9902a92b78abe..bbd75000acfcf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -194,7 +194,7 @@ public class GridCacheProcessor extends GridProcessorAdapter { private final Map stoppedCaches = new ConcurrentHashMap<>(); /** Map of proxies. */ - private final ConcurrentHashMap> jCacheProxies; + private final ConcurrentHashMap> jCacheProxies; /** Caches stop sequence. */ private final Deque stopSeq; @@ -948,7 +948,7 @@ public void stopCaches(boolean cancel) { */ public void blockGateways() { for (IgniteCacheProxy proxy : jCacheProxies.values()) - proxy.gate().onStopped(); + proxy.context().gate().onStopped(); } /** {@inheritDoc} */ @@ -1805,7 +1805,7 @@ void prepareCacheStart( CacheConfiguration ccfg = new CacheConfiguration(startCfg); - IgniteCacheProxy proxy = jCacheProxies.get(ccfg.getName()); + IgniteCacheProxyImpl proxy = jCacheProxies.get(ccfg.getName()); boolean proxyRestart = proxy != null && proxy.isRestarting() && !caches.containsKey(ccfg.getName()); @@ -1944,14 +1944,14 @@ private CacheGroupContext startCacheGroup( */ void blockGateway(String cacheName, boolean stop, boolean restart) { // Break the proxy before exchange future is done. - IgniteCacheProxy proxy = jCacheProxies.get(cacheName); + IgniteCacheProxyImpl proxy = jCacheProxies.get(cacheName); if (proxy != null) { if (stop) { if (restart) proxy.restart(); - proxy.gate().stopped(); + proxy.context().gate().stopped(); } else proxy.closeProxy(); @@ -1965,7 +1965,7 @@ void blockGateway(String cacheName, boolean stop, boolean restart) { private void stopGateway(DynamicCacheChangeRequest req) { assert req.stop() : req; - IgniteCacheProxy proxy; + IgniteCacheProxyImpl proxy; // Break the proxy before exchange future is done. if (req.restart()) { @@ -1978,7 +1978,7 @@ private void stopGateway(DynamicCacheChangeRequest req) { proxy = jCacheProxies.remove(req.cacheName()); if (proxy != null) - proxy.gate().onStopped(); + proxy.context().gate().onStopped(); } /** @@ -2014,7 +2014,7 @@ void initCacheProxies(AffinityTopologyVersion startTopVer, @Nullable Throwable e if (cacheCtx.startTopologyVersion().equals(startTopVer) ) { if (!jCacheProxies.containsKey(cacheCtx.name())) - jCacheProxies.putIfAbsent(cacheCtx.name(), new IgniteCacheProxy(cache.context(), cache, null, false)); + jCacheProxies.putIfAbsent(cacheCtx.name(), new IgniteCacheProxyImpl(cache.context(), cache, false)); if (cacheCtx.preloader() != null) cacheCtx.preloader().onInitialExchangeComplete(err); @@ -2079,7 +2079,7 @@ private void closeCache(GridCacheContext cctx, boolean destroy) { assert cache != null : cctx.name(); - jCacheProxies.put(cctx.name(), new IgniteCacheProxy(cache.context(), cache, null, false)); + jCacheProxies.put(cctx.name(), new IgniteCacheProxyImpl(cache.context(), cache, false)); } else { jCacheProxies.remove(cctx.name()); @@ -2129,6 +2129,18 @@ public void onExchangeDone( } for (ExchangeActions.CacheActionData action : exchActions.cacheStopRequests()) { + CacheGroupContext gctx = cacheGrps.get(action.descriptor().groupId()); + + // Cancel all operations blocking gateway + if (gctx != null) { + final String msg = "Failed to wait for topology update, cache group is stopping."; + + // If snapshot operation in progress we must throw CacheStoppedException + // for correct cache proxy restart. For more details see + // IgniteCacheProxy.cacheException() + gctx.affinity().cancelFutures(new CacheStoppedException(msg)); + } + stopGateway(action.request()); sharedCtx.database().checkpointReadLock(); @@ -2764,7 +2776,7 @@ IgniteInternalFuture dynamicCloseCache(String cacheName) { IgniteCacheProxy proxy = jCacheProxies.get(cacheName); - if (proxy == null || proxy.proxyClosed()) + if (proxy == null || proxy.isProxyClosed()) return new GridFinishedFuture<>(); // No-op. checkEmptyTransactions(); @@ -3202,7 +3214,11 @@ public IgniteInternalCache getOrStartCache(String name, CacheConfig * @return All configured cache instances. */ public Collection> jcaches() { - return jCacheProxies.values(); + return F.viewReadOnly(jCacheProxies.values(), new IgniteClosure, IgniteCacheProxy>() { + @Override public IgniteCacheProxy apply(IgniteCacheProxyImpl entry) { + return new GatewayProtectedCacheProxy<>(entry, new CacheOperationContext(), true); + } + }); } /** @@ -3226,7 +3242,7 @@ private IgniteInternalCache internalCacheEx(String name) { GridCacheAdapter cacheAdapter = caches.get(name); if (cacheAdapter != null) - proxy = new IgniteCacheProxy(cacheAdapter.context(), cacheAdapter, null, false); + proxy = new IgniteCacheProxyImpl(cacheAdapter.context(), cacheAdapter, false); } assert proxy != null : name; @@ -3301,13 +3317,16 @@ public IgniteCacheProxy publicJCache(String cacheName) throws Ignit IgniteCacheProxy cache = jCacheProxies.get(cacheName); + // Try to start cache, there is no guarantee that cache will be instantiated. if (cache == null) { dynamicStartCache(null, cacheName, null, false, failIfNotStarted, checkThreadTx).get(); cache = jCacheProxies.get(cacheName); } - return (IgniteCacheProxy)cache; + return cache != null ? + new GatewayProtectedCacheProxy<>((IgniteCacheProxy)cache, new CacheOperationContext(), true) : + null; } /** @@ -3424,7 +3443,7 @@ else if (ctx.clientDisconnected()) { public IgniteCacheProxy jcache(String name) { assert name != null; - IgniteCacheProxy cache = (IgniteCacheProxy)jCacheProxies.get(name); + IgniteCacheProxy cache = (IgniteCacheProxy) jCacheProxies.get(name); if (cache == null) throw new IllegalArgumentException("Cache is not configured: " + name); @@ -3446,9 +3465,9 @@ public IgniteCacheProxy jcache(String name) { public Collection> publicCaches() { Collection> res = new ArrayList<>(jCacheProxies.size()); - for (Map.Entry> entry : jCacheProxies.entrySet()) { + for (Map.Entry> entry : jCacheProxies.entrySet()) { if (entry.getValue().context().userCache()) - res.add(entry.getValue()); + res.add(new GatewayProtectedCacheProxy(entry.getValue(), new CacheOperationContext(), true)); } return res; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index 347e030d2ed1d..22bd676c69543 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -18,2483 +18,25 @@ package org.apache.ignite.internal.processors.cache; import java.io.Externalizable; -import java.io.IOException; -import java.io.ObjectInput; -import java.io.ObjectOutput; -import java.util.Collection; -import java.util.Collections; import java.util.Date; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; import java.util.UUID; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.Lock; -import javax.cache.Cache; -import javax.cache.CacheException; -import javax.cache.configuration.CacheEntryListenerConfiguration; -import javax.cache.configuration.Configuration; -import javax.cache.expiry.ExpiryPolicy; -import javax.cache.integration.CompletionListener; -import javax.cache.processor.EntryProcessor; -import javax.cache.processor.EntryProcessorException; -import javax.cache.processor.EntryProcessorResult; import org.apache.ignite.IgniteCache; -import org.apache.ignite.IgniteCacheRestartingException; -import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.IgniteException; -import org.apache.ignite.cache.CacheEntry; -import org.apache.ignite.cache.CacheEntryProcessor; -import org.apache.ignite.cache.CacheManager; -import org.apache.ignite.cache.CacheMetrics; -import org.apache.ignite.cache.CachePeekMode; -import org.apache.ignite.cache.query.ContinuousQuery; -import org.apache.ignite.cache.query.FieldsQueryCursor; -import org.apache.ignite.cache.query.Query; -import org.apache.ignite.cache.query.QueryCursor; -import org.apache.ignite.cache.query.QueryDetailMetrics; -import org.apache.ignite.cache.query.QueryMetrics; -import org.apache.ignite.cache.query.ScanQuery; -import org.apache.ignite.cache.query.SpiQuery; -import org.apache.ignite.cache.query.SqlFieldsQuery; -import org.apache.ignite.cache.query.SqlQuery; -import org.apache.ignite.cache.query.TextQuery; -import org.apache.ignite.cluster.ClusterGroup; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.internal.AsyncSupportAdapter; -import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.IgniteInternalFuture; -import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; -import org.apache.ignite.internal.processors.cache.query.CacheQuery; -import org.apache.ignite.internal.processors.cache.query.CacheQueryFuture; -import org.apache.ignite.internal.processors.cache.query.GridCacheQueryType; -import org.apache.ignite.internal.processors.query.QueryUtils; -import org.apache.ignite.internal.util.GridCloseableIteratorAdapter; -import org.apache.ignite.internal.util.GridEmptyIterator; -import org.apache.ignite.internal.util.future.GridFutureAdapter; -import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl; -import org.apache.ignite.internal.util.future.IgniteFutureImpl; -import org.apache.ignite.internal.util.lang.GridCloseableIterator; -import org.apache.ignite.internal.util.lang.GridClosureException; -import org.apache.ignite.internal.util.lang.IgniteOutClosureX; -import org.apache.ignite.internal.util.tostring.GridToStringExclude; -import org.apache.ignite.internal.util.tostring.GridToStringInclude; -import org.apache.ignite.internal.util.typedef.CI1; -import org.apache.ignite.internal.util.typedef.CX1; -import org.apache.ignite.internal.util.typedef.internal.A; -import org.apache.ignite.internal.util.typedef.internal.CU; -import org.apache.ignite.internal.util.typedef.internal.S; -import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.lang.IgniteBiPredicate; -import org.apache.ignite.lang.IgniteClosure; +import org.apache.ignite.lang.IgniteAsyncSupport; import org.apache.ignite.lang.IgniteFuture; -import org.apache.ignite.lang.IgniteInClosure; -import org.apache.ignite.mxbean.CacheMetricsMXBean; -import org.apache.ignite.plugin.security.SecurityPermission; -import org.jetbrains.annotations.Nullable; - -/** - * Cache proxy. - */ -@SuppressWarnings("unchecked") -public class IgniteCacheProxy extends AsyncSupportAdapter> - implements IgniteCache, Externalizable { - /** */ - private static final long serialVersionUID = 0L; - - /** Context. */ - private volatile GridCacheContext ctx; - - /** Gateway. */ - private volatile GridCacheGateway gate; - - /** Delegate. */ - @GridToStringInclude - private volatile IgniteInternalCache delegate; - - /** Operation context. */ - private CacheOperationContext opCtx; - - /** */ - @GridToStringExclude - private GridCacheProxyImpl internalProxy; - - /** */ - @GridToStringExclude - private CacheManager cacheMgr; - - /** If {@code false} does not acquire read lock on gateway enter. */ - @GridToStringExclude - private boolean lock; - - /** */ - private final AtomicReference> restartFut = new AtomicReference<>(null); - - /** - * Empty constructor required for {@link Externalizable}. - */ - public IgniteCacheProxy() { - // No-op. - } - - /** - * @param ctx Context. - * @param delegate Delegate. - * @param opCtx Operation context. - * @param async Async support flag. - */ - public IgniteCacheProxy( - GridCacheContext ctx, - IgniteInternalCache delegate, - CacheOperationContext opCtx, - boolean async - ) { - this(ctx, delegate, opCtx, async, true); - } - - /** - * @param ctx Context. - * @param delegate Delegate. - * @param opCtx Operation context. - * @param async Async support flag. - * @param lock If {@code false} does not acquire read lock on gateway enter. - */ - private IgniteCacheProxy( - GridCacheContext ctx, - IgniteInternalCache delegate, - @Nullable CacheOperationContext opCtx, - boolean async, - boolean lock - ) { - super(async); - - assert ctx != null; - assert delegate != null; - - this.ctx = ctx; - this.delegate = delegate; - this.opCtx = opCtx; - - gate = ctx.gate(); - - internalProxy = new GridCacheProxyImpl<>(ctx, delegate, opCtx); - - this.lock = lock; - } - - /** - * @return Operation context. - */ - @Nullable public CacheOperationContext operationContext() { - return opCtx; - } - - /** - * Gets cache proxy which does not acquire read lock on gateway enter, should be used only if grid read lock is - * externally acquired. - * - * @return Ignite cache proxy with simple gate. - */ - public IgniteCacheProxy cacheNoGate() { - return new IgniteCacheProxy<>(ctx, delegate, opCtx, isAsync(), false); - } - - /** - * @return Context. - */ - public GridCacheContext context() { - return ctx; - } - - /** - * @return Gateway. - */ - public GridCacheGateway gate() { - return gate; - } - - /** {@inheritDoc} */ - @Override public CacheMetrics metrics() { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return ctx.cache().clusterMetrics(); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public CacheMetrics metrics(ClusterGroup grp) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return ctx.cache().clusterMetrics(grp); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public CacheMetrics localMetrics() { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return ctx.cache().localMetrics(); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public CacheMetricsMXBean mxBean() { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return ctx.cache().clusterMxBean(); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public CacheMetricsMXBean localMxBean() { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return ctx.cache().localMxBean(); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public > C getConfiguration(Class clazz) { - CacheConfiguration cfg = ctx.config(); - - if (!clazz.isAssignableFrom(cfg.getClass())) - throw new IllegalArgumentException(); - - return clazz.cast(cfg); - } - - /** {@inheritDoc} */ - @Override public IgniteCache withExpiryPolicy(ExpiryPolicy plc) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - CacheOperationContext prj0 = opCtx != null ? opCtx.withExpiryPolicy(plc) : - new CacheOperationContext(false, null, false, plc, false, null, false); - - return new IgniteCacheProxy<>(ctx, delegate, prj0, isAsync(), lock); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public IgniteCache withSkipStore() { - return skipStore(); - } - - /** {@inheritDoc} */ - @Override public IgniteCache withKeepBinary() { - return keepBinary(); - } - - /** {@inheritDoc} */ - @Override public IgniteCache withNoRetries() { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - boolean noRetries = opCtx != null && opCtx.noRetries(); - - if (noRetries) - return this; - - CacheOperationContext opCtx0 = opCtx != null ? opCtx.setNoRetries(true) : - new CacheOperationContext(false, null, false, null, true, null, false); - - return new IgniteCacheProxy<>(ctx, - delegate, - opCtx0, - isAsync(), - lock); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public IgniteCache withPartitionRecover() { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - boolean recovery = opCtx != null && opCtx.recovery(); - - if (recovery) - return this; - - CacheOperationContext opCtx0 = opCtx != null ? opCtx.setRecovery(true) : - new CacheOperationContext(false, null, false, null, false, null, true); - - return new IgniteCacheProxy<>(ctx, - delegate, - opCtx0, - isAsync(), - lock); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public void loadCache(@Nullable IgniteBiPredicate p, @Nullable Object... args) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - if (ctx.cache().isLocal()) - setFuture(ctx.cache().localLoadCacheAsync(p, args)); - else - setFuture(ctx.cache().globalLoadCacheAsync(p, args)); - } - else { - if (ctx.cache().isLocal()) - ctx.cache().localLoadCache(p, args); - else - ctx.cache().globalLoadCache(p, args); - } - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture loadCacheAsync(@Nullable IgniteBiPredicate p, - @Nullable Object... args) throws CacheException { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (ctx.cache().isLocal()) - return (IgniteFuture)createFuture(ctx.cache().localLoadCacheAsync(p, args)); - else - return (IgniteFuture)createFuture(ctx.cache().globalLoadCacheAsync(p, args)); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public void localLoadCache(@Nullable IgniteBiPredicate p, @Nullable Object... args) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) - setFuture(delegate.localLoadCacheAsync(p, args)); - else - delegate.localLoadCache(p, args); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture localLoadCacheAsync(@Nullable IgniteBiPredicate p, - @Nullable Object... args) throws CacheException { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return (IgniteFuture)createFuture(delegate.localLoadCacheAsync(p, args)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Nullable @Override public V getAndPutIfAbsent(K key, V val) throws CacheException { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(delegate.getAndPutIfAbsentAsync(key, val)); - - return null; - } - else - return delegate.getAndPutIfAbsent(key, val); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture getAndPutIfAbsentAsync(K key, V val) throws CacheException { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(delegate.getAndPutIfAbsentAsync(key, val)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public Lock lock(K key) throws CacheException { - return lockAll(Collections.singleton(key)); - } - - /** {@inheritDoc} */ - @Override public Lock lockAll(final Collection keys) { - return new CacheLockImpl<>(gate, delegate, opCtx, keys); - } - - /** {@inheritDoc} */ - @Override public boolean isLocalLocked(K key, boolean byCurrThread) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return byCurrThread ? delegate.isLockedByThread(key) : delegate.isLocked(key); - } - finally { - onLeave(gate, prev); - } - } - - /** - * @param scanQry ScanQry. - * @param transformer Transformer - * @param grp Optional cluster group. - * @return Cursor. - * @throws IgniteCheckedException If failed. - */ - @SuppressWarnings("unchecked") - private QueryCursor query( - final ScanQuery scanQry, - @Nullable final IgniteClosure transformer, - @Nullable ClusterGroup grp) - throws IgniteCheckedException { - - final CacheQuery qry; - - boolean isKeepBinary = opCtx != null && opCtx.isKeepBinary(); - - IgniteBiPredicate p = scanQry.getFilter(); - - qry = ctx.queries().createScanQuery(p, transformer, scanQry.getPartition(), isKeepBinary); - - if (scanQry.getPageSize() > 0) - qry.pageSize(scanQry.getPageSize()); - - if (grp != null) - qry.projection(grp); - - final GridCloseableIterator iter = ctx.kernalContext().query().executeQuery(GridCacheQueryType.SCAN, - ctx.name(), ctx, new IgniteOutClosureX>() { - @Override public GridCloseableIterator applyx() throws IgniteCheckedException { - final GridCloseableIterator iter0 = qry.executeScanQuery(); - - final boolean needToConvert = transformer == null; - - return new GridCloseableIteratorAdapter() { - @Override protected R onNext() throws IgniteCheckedException { - Object next = iter0.nextX(); - - if (needToConvert) { - Map.Entry entry = (Map.Entry)next; - - return (R)new CacheEntryImpl<>(entry.getKey(), entry.getValue()); - } - - return (R)next; - } - - @Override protected boolean onHasNext() throws IgniteCheckedException { - return iter0.hasNextX(); - } - - @Override protected void onClose() throws IgniteCheckedException { - iter0.close(); - } - }; - } - }, true); - - return new QueryCursorImpl<>(iter); - } - - /** - * @param filter Filter. - * @param grp Optional cluster group. - * @return Cursor. - * @throws IgniteCheckedException If failed. - */ - @SuppressWarnings("unchecked") - private QueryCursor> query(final Query filter, @Nullable ClusterGroup grp) - throws IgniteCheckedException { - final CacheQuery qry; - - boolean isKeepBinary = opCtx != null && opCtx.isKeepBinary(); - - final CacheQueryFuture fut; - - if (filter instanceof TextQuery) { - TextQuery p = (TextQuery)filter; - - qry = ctx.queries().createFullTextQuery(p.getType(), p.getText(), isKeepBinary); - - if (grp != null) - qry.projection(grp); - - fut = ctx.kernalContext().query().executeQuery(GridCacheQueryType.TEXT, p.getText(), ctx, - new IgniteOutClosureX>>() { - @Override public CacheQueryFuture> applyx() { - return qry.execute(); - } - }, false); - } - else if (filter instanceof SpiQuery) { - qry = ctx.queries().createSpiQuery(isKeepBinary); - - if (grp != null) - qry.projection(grp); - - fut = ctx.kernalContext().query().executeQuery(GridCacheQueryType.SPI, filter.getClass().getSimpleName(), - ctx, new IgniteOutClosureX>>() { - @Override public CacheQueryFuture> applyx() { - return qry.execute(((SpiQuery)filter).getArgs()); - } - }, false); - } - else { - if (filter instanceof SqlFieldsQuery) - throw new CacheException("Use methods 'queryFields' and 'localQueryFields' for " + - SqlFieldsQuery.class.getSimpleName() + "."); - - throw new CacheException("Unsupported query type: " + filter); - } - - return new QueryCursorImpl<>(new GridCloseableIteratorAdapter>() { - /** */ - private Cache.Entry cur; - - @Override protected Entry onNext() throws IgniteCheckedException { - if (!onHasNext()) - throw new NoSuchElementException(); - - Cache.Entry e = cur; - - cur = null; - - return e; - } - - @Override protected boolean onHasNext() throws IgniteCheckedException { - if (cur != null) - return true; - - Object next = fut.next(); - - // Workaround a bug: if IndexingSpi is configured future represents Iterator - // instead of Iterator due to IndexingSpi interface. - if (next == null) - return false; - - if (next instanceof Cache.Entry) - cur = (Cache.Entry)next; - else { - Map.Entry e = (Map.Entry)next; - - cur = new CacheEntryImpl(e.getKey(), e.getValue()); - } - - return true; - } - - @Override protected void onClose() throws IgniteCheckedException { - fut.cancel(); - } - }); - } - - /** - * @param loc Enforce local. - * @return Local node cluster group. - */ - private ClusterGroup projection(boolean loc) { - if (loc || ctx.isLocal() || ctx.isReplicatedAffinityNode()) - return ctx.kernalContext().grid().cluster().forLocal(); - - if (ctx.isReplicated()) - return ctx.kernalContext().grid().cluster().forDataNodes(ctx.name()).forRandom(); - - return null; - } - - /** - * Executes continuous query. - * - * @param qry Query. - * @param loc Local flag. - * @param keepBinary Keep binary flag. - * @return Initial iteration cursor. - */ - @SuppressWarnings("unchecked") - private QueryCursor> queryContinuous(ContinuousQuery qry, boolean loc, boolean keepBinary) { - if (qry.getInitialQuery() instanceof ContinuousQuery) - throw new IgniteException("Initial predicate for continuous query can't be an instance of another " + - "continuous query. Use SCAN or SQL query for initial iteration."); - - if (qry.getLocalListener() == null) - throw new IgniteException("Mandatory local listener is not set for the query: " + qry); - - if (qry.getRemoteFilter() != null && qry.getRemoteFilterFactory() != null) - throw new IgniteException("Should be used either RemoterFilter or RemoteFilterFactory."); - - try { - final UUID routineId = ctx.continuousQueries().executeQuery( - qry.getLocalListener(), - qry.getRemoteFilter(), - qry.getRemoteFilterFactory(), - qry.getPageSize(), - qry.getTimeInterval(), - qry.isAutoUnsubscribe(), - loc, - keepBinary, - qry.isIncludeExpired()); - - final QueryCursor> cur = - qry.getInitialQuery() != null ? query(qry.getInitialQuery()) : null; - - return new QueryCursor>() { - @Override public Iterator> iterator() { - return cur != null ? cur.iterator() : new GridEmptyIterator>(); - } - - @Override public List> getAll() { - return cur != null ? cur.getAll() : Collections.>emptyList(); - } - - @Override public void close() { - if (cur != null) - cur.close(); - - try { - ctx.kernalContext().continuous().stopRoutine(routineId).get(); - } - catch (IgniteCheckedException e) { - throw U.convertException(e); - } - } - }; - } - catch (IgniteCheckedException e) { - throw U.convertException(e); - } - } - - /** {@inheritDoc} */ - @SuppressWarnings("unchecked") - @Override public FieldsQueryCursor> query(SqlFieldsQuery qry) { - return (FieldsQueryCursor>)query((Query)qry); - } - - /** {@inheritDoc} */ - @SuppressWarnings("unchecked") - @Override public QueryCursor query(Query qry) { - A.notNull(qry, "qry"); - - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - ctx.checkSecurity(SecurityPermission.CACHE_READ); - - validate(qry); - - convertToBinary(qry); - - CacheOperationContext opCtxCall = ctx.operationContextPerCall(); - - boolean keepBinary = opCtxCall != null && opCtxCall.isKeepBinary(); - - if (qry instanceof ContinuousQuery) - return (QueryCursor)queryContinuous((ContinuousQuery)qry, qry.isLocal(), keepBinary); - - if (qry instanceof SqlQuery) - return (QueryCursor)ctx.kernalContext().query().querySql(ctx, (SqlQuery)qry, keepBinary); - - if (qry instanceof SqlFieldsQuery) - return (FieldsQueryCursor)ctx.kernalContext().query().querySqlFields(ctx, (SqlFieldsQuery)qry, - keepBinary); - - if (qry instanceof ScanQuery) - return query((ScanQuery)qry, null, projection(qry.isLocal())); - - return (QueryCursor)query(qry, projection(qry.isLocal())); - } - catch (Exception e) { - if (e instanceof CacheException) - throw (CacheException)e; - - throw new CacheException(e); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public QueryCursor query(Query qry, IgniteClosure transformer) { - A.notNull(qry, "qry"); - A.notNull(transformer, "transformer"); - - if (!(qry instanceof ScanQuery)) - throw new UnsupportedOperationException("Transformers are supported only for SCAN queries."); - - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - ctx.checkSecurity(SecurityPermission.CACHE_READ); - - validate(qry); - - return query((ScanQuery)qry, transformer, projection(qry.isLocal())); - } - catch (Exception e) { - if (e instanceof CacheException) - throw (CacheException)e; - - throw new CacheException(e); - } - finally { - onLeave(gate, prev); - } - } - - /** - * Convert query arguments to BinaryObjects if binary marshaller used. - * - * @param qry Query. - */ - private void convertToBinary(final Query qry) { - if (ctx.binaryMarshaller()) { - if (qry instanceof SqlQuery) { - final SqlQuery sqlQry = (SqlQuery) qry; - - convertToBinary(sqlQry.getArgs()); - } - else if (qry instanceof SpiQuery) { - final SpiQuery spiQry = (SpiQuery) qry; - - convertToBinary(spiQry.getArgs()); - } - else if (qry instanceof SqlFieldsQuery) { - final SqlFieldsQuery fieldsQry = (SqlFieldsQuery) qry; - - convertToBinary(fieldsQry.getArgs()); - } - } - } - - /** - * Converts query arguments to BinaryObjects if binary marshaller used. - * - * @param args Arguments. - */ - private void convertToBinary(final Object[] args) { - if (args == null) - return; - - for (int i = 0; i < args.length; i++) - args[i] = ctx.cacheObjects().binary().toBinary(args[i]); - } - - /** - * Checks query. - * - * @param qry Query - * @throws CacheException If query indexing disabled for sql query. - */ - private void validate(Query qry) { - if (!QueryUtils.isEnabled(ctx.config()) && !(qry instanceof ScanQuery) && - !(qry instanceof ContinuousQuery) && !(qry instanceof SpiQuery) && !(qry instanceof SqlQuery) && - !(qry instanceof SqlFieldsQuery)) - throw new CacheException("Indexing is disabled for cache: " + ctx.cache().name() + - ". Use setIndexedTypes or setTypeMetadata methods on CacheConfiguration to enable."); - - if (!ctx.kernalContext().query().moduleEnabled() && - (qry instanceof SqlQuery || qry instanceof SqlFieldsQuery || qry instanceof TextQuery)) - throw new CacheException("Failed to execute query. Add module 'ignite-indexing' to the classpath " + - "of all Ignite nodes."); - } - - /** {@inheritDoc} */ - @Override public Iterable> localEntries(CachePeekMode... peekModes) throws CacheException { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return delegate.localEntries(peekModes); - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public QueryMetrics queryMetrics() { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return delegate.context().queries().metrics(); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public void resetQueryMetrics() { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - delegate.context().queries().resetMetrics(); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public Collection queryDetailMetrics() { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return delegate.context().queries().detailMetrics(); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public void resetQueryDetailMetrics() { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - delegate.context().queries().resetDetailMetrics(); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public void localEvict(Collection keys) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - delegate.evictAll(keys); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Nullable @Override public V localPeek(K key, CachePeekMode... peekModes) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return delegate.localPeek(key, peekModes, null); - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public int size(CachePeekMode... peekModes) throws CacheException { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(delegate.sizeAsync(peekModes)); - - return 0; - } - else - return delegate.size(peekModes); - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture sizeAsync(CachePeekMode... peekModes) throws CacheException { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(delegate.sizeAsync(peekModes)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public long sizeLong(CachePeekMode... peekModes) throws CacheException { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(delegate.sizeLongAsync(peekModes)); - - return 0; - } - else - return delegate.sizeLong(peekModes); - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture sizeLongAsync(CachePeekMode... peekModes) throws CacheException { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(delegate.sizeLongAsync(peekModes)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public long sizeLong(int part, CachePeekMode... peekModes) throws CacheException { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(delegate.sizeLongAsync(part, peekModes)); - - return 0; - } - else - return delegate.sizeLong(part, peekModes); - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture sizeLongAsync(int part, CachePeekMode... peekModes) throws CacheException { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(delegate.sizeLongAsync(part, peekModes)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public int localSize(CachePeekMode... peekModes) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return delegate.localSize(peekModes); - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public long localSizeLong(CachePeekMode... peekModes) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return delegate.localSizeLong(peekModes); - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public long localSizeLong(int part, CachePeekMode... peekModes) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return delegate.localSizeLong(part, peekModes); - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public V get(K key) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(delegate.getAsync(key)); - - return null; - } - else - return delegate.get(key); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture getAsync(K key) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(delegate.getAsync(key)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public CacheEntry getEntry(K key) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(delegate.getEntryAsync(key)); - - return null; - } - else - return delegate.getEntry(key); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture> getEntryAsync(K key) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(delegate.getEntryAsync(key)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public Map getAll(Set keys) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(delegate.getAllAsync(keys)); - - return null; - } - else - return delegate.getAll(keys); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture> getAllAsync(Set keys) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(delegate.getAllAsync(keys)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public Collection> getEntries(Set keys) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(delegate.getEntriesAsync(keys)); - - return null; - } - else - return delegate.getEntries(keys); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture>> getEntriesAsync(Set keys) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(delegate.getEntriesAsync(keys)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public Map getAllOutTx(Set keys) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(delegate.getAllOutTxAsync(keys)); - - return null; - } - else - return delegate.getAllOutTx(keys); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture> getAllOutTxAsync(Set keys) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(delegate.getAllOutTxAsync(keys)); - } - finally { - onLeave(gate, prev); - } - } - - /** - * @param keys Keys. - * @return Values map. - */ - public Map getAll(Collection keys) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(delegate.getAllAsync(keys)); - - return null; - } - else - return delegate.getAll(keys); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public boolean containsKey(K key) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(delegate.containsKeyAsync(key)); - - return false; - } - else - return delegate.containsKey(key); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture containsKeyAsync(K key) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(delegate.containsKeyAsync(key)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public boolean containsKeys(Set keys) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(delegate.containsKeysAsync(keys)); - - return false; - } - else - return delegate.containsKeys(keys); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture containsKeysAsync(Set keys) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(delegate.containsKeysAsync(keys)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public void loadAll( - Set keys, - boolean replaceExisting, - @Nullable final CompletionListener completionLsnr - ) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - IgniteInternalFuture fut = ctx.cache().loadAll(keys, replaceExisting); - - if (completionLsnr != null) { - fut.listen(new CI1>() { - @Override public void apply(IgniteInternalFuture fut) { - try { - fut.get(); - - completionLsnr.onCompletion(); - } - catch (IgniteCheckedException e) { - completionLsnr.onException(cacheException(e)); - } - } - }); - } - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public void put(K key, V val) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) - setFuture(putAsync0(key, val)); - else - delegate.put(key, val); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture putAsync(K key, V val) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(putAsync0(key, val)); - } - finally { - onLeave(gate, prev); - } - } - - /** - * Put async internal operation implementation. - * - * @param key Key. - * @param val Value. - * @return Internal future. - */ - private IgniteInternalFuture putAsync0(K key, V val) { - IgniteInternalFuture fut = delegate.putAsync(key, val); - - return fut.chain(new CX1, Void>() { - @Override public Void applyx(IgniteInternalFuture fut1) throws IgniteCheckedException { - try { - fut1.get(); - } - catch (RuntimeException e) { - throw new GridClosureException(e); - } - - return null; - } - }); - } - - /** {@inheritDoc} */ - @Override public V getAndPut(K key, V val) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(delegate.getAndPutAsync(key, val)); - - return null; - } - else - return delegate.getAndPut(key, val); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture getAndPutAsync(K key, V val) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(delegate.getAndPutAsync(key, val)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public void putAll(Map map) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) - setFuture(delegate.putAllAsync(map)); - else - delegate.putAll(map); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture putAllAsync(Map map) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return (IgniteFuture)createFuture(delegate.putAllAsync(map)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public boolean putIfAbsent(K key, V val) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(delegate.putIfAbsentAsync(key, val)); - - return false; - } - else - return delegate.putIfAbsent(key, val); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture putIfAbsentAsync(K key, V val) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(delegate.putIfAbsentAsync(key, val)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public boolean remove(K key) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(delegate.removeAsync(key)); - - return false; - } - else - return delegate.remove(key); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture removeAsync(K key) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(delegate.removeAsync(key)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public boolean remove(K key, V oldVal) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(delegate.removeAsync(key, oldVal)); - - return false; - } - else - return delegate.remove(key, oldVal); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture removeAsync(K key, V oldVal) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(delegate.removeAsync(key, oldVal)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public V getAndRemove(K key) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(delegate.getAndRemoveAsync(key)); - - return null; - } - else - return delegate.getAndRemove(key); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture getAndRemoveAsync(K key) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(delegate.getAndRemoveAsync(key)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public boolean replace(K key, V oldVal, V newVal) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(delegate.replaceAsync(key, oldVal, newVal)); - - return false; - } - else - return delegate.replace(key, oldVal, newVal); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture replaceAsync(K key, V oldVal, V newVal) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(delegate.replaceAsync(key, oldVal, newVal)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public boolean replace(K key, V val) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(delegate.replaceAsync(key, val)); - - return false; - } - else - return delegate.replace(key, val); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture replaceAsync(K key, V val) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(delegate.replaceAsync(key, val)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public V getAndReplace(K key, V val) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(delegate.getAndReplaceAsync(key, val)); - - return null; - } - else - return delegate.getAndReplace(key, val); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture getAndReplaceAsync(K key, V val) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(delegate.getAndReplaceAsync(key, val)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public void removeAll(Set keys) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) - setFuture(delegate.removeAllAsync(keys)); - else - delegate.removeAll(keys); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture removeAllAsync(Set keys) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return (IgniteFuture)createFuture(delegate.removeAllAsync(keys)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public void removeAll() { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) - setFuture(delegate.removeAllAsync()); - else - delegate.removeAll(); - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture removeAllAsync() { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return (IgniteFuture)createFuture(delegate.removeAllAsync()); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public void clear(K key) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) - setFuture(delegate.clearAsync(key)); - else - delegate.clear(key); - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture clearAsync(K key) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return (IgniteFuture)createFuture(delegate.clearAsync(key)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public void clearAll(Set keys) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) - setFuture(delegate.clearAllAsync(keys)); - else - delegate.clearAll(keys); - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture clearAllAsync(Set keys) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return (IgniteFuture)createFuture(delegate.clearAllAsync(keys)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public void clear() { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) - setFuture(delegate.clearAsync()); - else - delegate.clear(); - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture clearAsync() { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return (IgniteFuture)createFuture(delegate.clearAsync()); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public void localClear(K key) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - delegate.clearLocally(key); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public void localClearAll(Set keys) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - for (K key : keys) - delegate.clearLocally(key); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public T invoke(K key, EntryProcessor entryProcessor, Object... args) - throws EntryProcessorException { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(invokeAsync0(key, entryProcessor, args)); - - return null; - } - else { - EntryProcessorResult res = delegate.invoke(key, entryProcessor, args); - - return res != null ? res.get() : null; - } - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture invokeAsync(K key, EntryProcessor entryProcessor, - Object... args) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(invokeAsync0(key, entryProcessor, args)); - } - finally { - onLeave(gate, prev); - } - } - - /** - * Invoke async operation internal implementation. - * - * @param key Key. - * @param entryProcessor Processor. - * @param args Arguments. - * @return Internal future. - */ - private IgniteInternalFuture invokeAsync0(K key, EntryProcessor entryProcessor, Object[] args) { - IgniteInternalFuture> fut = delegate.invokeAsync(key, entryProcessor, args); - - return fut.chain(new CX1>, T>() { - @Override public T applyx(IgniteInternalFuture> fut1) - throws IgniteCheckedException { - try { - EntryProcessorResult res = fut1.get(); - - return res != null ? res.get() : null; - } - catch (RuntimeException e) { - throw new GridClosureException(e); - } - } - }); - } - - - /** {@inheritDoc} */ - @Override public T invoke(K key, CacheEntryProcessor entryProcessor, Object... args) - throws EntryProcessorException { - return invoke(key, (EntryProcessor)entryProcessor, args); - } - - /** {@inheritDoc} */ - @Override public IgniteFuture invokeAsync(K key, CacheEntryProcessor entryProcessor, - Object... args) { - return invokeAsync(key, (EntryProcessor)entryProcessor, args); - } - - /** - * @param topVer Locked topology version. - * @param key Key. - * @param entryProcessor Entry processor. - * @param args Arguments. - * @return Invoke result. - */ - public T invoke(@Nullable AffinityTopologyVersion topVer, - K key, - EntryProcessor entryProcessor, - Object... args) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) - throw new UnsupportedOperationException(); - else { - EntryProcessorResult res = delegate.invoke(topVer, key, entryProcessor, args); - - return res != null ? res.get() : null; - } - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public Map> invokeAll(Set keys, - EntryProcessor entryProcessor, - Object... args) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(delegate.invokeAllAsync(keys, entryProcessor, args)); - - return null; - } - else - return delegate.invokeAll(keys, entryProcessor, args); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture>> invokeAllAsync(Set keys, - EntryProcessor entryProcessor, Object... args) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(delegate.invokeAllAsync(keys, entryProcessor, args)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public Map> invokeAll(Set keys, - CacheEntryProcessor entryProcessor, - Object... args) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(delegate.invokeAllAsync(keys, entryProcessor, args)); - - return null; - } - else - return delegate.invokeAll(keys, entryProcessor, args); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture>> invokeAllAsync(Set keys, - CacheEntryProcessor entryProcessor, Object... args) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(delegate.invokeAllAsync(keys, entryProcessor, args)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public Map> invokeAll( - Map> map, - Object... args) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - if (isAsync()) { - setFuture(delegate.invokeAllAsync(map, args)); - - return null; - } - else - return delegate.invokeAll(map, args); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public IgniteFuture>> invokeAllAsync( - Map> map, Object... args) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return createFuture(delegate.invokeAllAsync(map, args)); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public String getName() { - return delegate.name(); - } - - /** {@inheritDoc} */ - @Override public CacheManager getCacheManager() { - return cacheMgr; - } +public interface IgniteCacheProxy extends IgniteCache, Externalizable { /** - * @param cacheMgr Cache manager. + * @return Context. */ - public void setCacheManager(CacheManager cacheMgr) { - this.cacheMgr = cacheMgr; - } - - /** {@inheritDoc} */ - @Override public void destroy() { - GridCacheGateway gate = this.gate; - - if (!onEnterIfNoStop(gate)) - return; - - IgniteInternalFuture fut; - - try { - fut = ctx.kernalContext().cache().dynamicDestroyCache(ctx.name(), false, true, false); - } - finally { - onLeave(gate); - } - - try { - fut.get(); - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public void close() { - GridCacheGateway gate = this.gate; - - if (!onEnterIfNoStop(gate)) - return; - - IgniteInternalFuture fut; - - try { - fut = ctx.kernalContext().cache().dynamicCloseCache(ctx.name()); - } - finally { - onLeave(gate); - } - - try { - fut.get(); - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** {@inheritDoc} */ - @Override public boolean isClosed() { - GridCacheGateway gate = this.gate; - - if (!onEnterIfNoStop(gate)) - return true; - - try { - return ctx.kernalContext().cache().context().closed(ctx); - } - finally { - onLeave(gate); - } - } + public GridCacheContext context(); /** - * @return Proxy delegate. + * Gets cache proxy which does not acquire read lock on gateway enter, should be used only if grid read lock is + * externally acquired. + * + * @return Ignite cache proxy with simple gate. */ - public IgniteInternalCache delegate() { - return delegate; - } - - /** {@inheritDoc} */ - @SuppressWarnings("unchecked") - @Override public T unwrap(Class clazz) { - if (clazz.isAssignableFrom(getClass())) - return (T)this; - else if (clazz.isAssignableFrom(IgniteEx.class)) - return (T)ctx.grid(); - - throw new IllegalArgumentException("Unwrapping to class is not supported: " + clazz); - } - - /** {@inheritDoc} */ - @Override public void registerCacheEntryListener(CacheEntryListenerConfiguration lsnrCfg) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - CacheOperationContext opCtx = ctx.operationContextPerCall(); - - ctx.continuousQueries().executeJCacheQuery(lsnrCfg, false, opCtx != null && opCtx.isKeepBinary()); - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public void deregisterCacheEntryListener(CacheEntryListenerConfiguration lsnrCfg) { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - ctx.continuousQueries().cancelJCacheQuery(lsnrCfg); - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public Iterator> iterator() { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return ctx.cache().igniteIterator(); - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override protected IgniteCache createAsyncInstance() { - return new IgniteCacheProxy<>(ctx, delegate, opCtx, true, lock); - } + public IgniteCacheProxy cacheNoGate(); /** * Creates projection that will operate with binary objects.

Projection returned by this method will force @@ -2520,368 +62,50 @@ else if (clazz.isAssignableFrom(IgniteEx.class)) * @return Projection for binary objects. */ @SuppressWarnings("unchecked") - public IgniteCache keepBinary() { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - CacheOperationContext opCtx0 = - new CacheOperationContext( - opCtx != null && opCtx.skipStore(), - opCtx != null ? opCtx.subjectId() : null, - true, - opCtx != null ? opCtx.expiry() : null, - opCtx != null && opCtx.noRetries(), - opCtx != null ? opCtx.dataCenterId() : null, - opCtx != null && opCtx.recovery()); - - return new IgniteCacheProxy<>((GridCacheContext)ctx, - (IgniteInternalCache)delegate, - opCtx0, - isAsync(), - lock); - } - finally { - onLeave(gate, prev); - } - } + public IgniteCache keepBinary(); /** * @param dataCenterId Data center ID. * @return Projection for data center id. */ @SuppressWarnings("unchecked") - public IgniteCache withDataCenterId(byte dataCenterId) { - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - Byte prevDataCenterId = opCtx != null ? opCtx.dataCenterId() : null; - - if (prevDataCenterId != null && dataCenterId == prevDataCenterId) - return this; - - CacheOperationContext opCtx0 = - new CacheOperationContext( - opCtx != null && opCtx.skipStore(), - opCtx != null ? opCtx.subjectId() : null, - opCtx != null && opCtx.isKeepBinary(), - opCtx != null ? opCtx.expiry() : null, - opCtx != null && opCtx.noRetries(), - dataCenterId, - opCtx != null && opCtx.recovery()); - - return new IgniteCacheProxy<>(ctx, - delegate, - opCtx0, - isAsync(), - lock); - } - finally { - onLeave(gate, prev); - } - } + public IgniteCache withDataCenterId(byte dataCenterId); /** * @return Cache with skip store enabled. */ - public IgniteCache skipStore() { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - boolean skip = opCtx != null && opCtx.skipStore(); - - if (skip) - return this; - - CacheOperationContext opCtx0 = - new CacheOperationContext(true, - opCtx != null ? opCtx.subjectId() : null, - opCtx != null && opCtx.isKeepBinary(), - opCtx != null ? opCtx.expiry() : null, - opCtx != null && opCtx.noRetries(), - opCtx != null ? opCtx.dataCenterId() : null, - opCtx != null && opCtx.recovery()); - - return new IgniteCacheProxy<>(ctx, - delegate, - opCtx0, - isAsync(), - lock); - } - finally { - onLeave(gate, prev); - } - } - - /** - * @param e Checked exception. - * @return Cache exception. - */ - private RuntimeException cacheException(IgniteCheckedException e) { - GridFutureAdapter restartFut = this.restartFut.get(); - - if (restartFut != null && !restartFut.isDone() && e.hasCause(CacheStoppedException.class)) - throw new IgniteCacheRestartingException(new IgniteFutureImpl<>(restartFut), "Cache is restarting: " + - ctx.name()); - - return CU.convertToCacheException(e); - } - - /** - * @param fut Future for async operation. - */ - private void setFuture(IgniteInternalFuture fut) { - curFut.set(createFuture(fut)); - } - - /** {@inheritDoc} */ - @Override protected IgniteFuture createFuture(IgniteInternalFuture fut) { - return new IgniteCacheFutureImpl<>(fut); - } + public IgniteCache skipStore(); /** * @return Internal proxy. */ - public GridCacheProxyImpl internalProxy() { - return internalProxy; - } + public GridCacheProxyImpl internalProxy(); /** * @return {@code True} if proxy was closed. */ - public boolean proxyClosed() { - return !gate.getClass().equals(GridCacheGateway.class); - } + public boolean isProxyClosed(); /** * Closes this proxy instance. */ - public void closeProxy() { - gate = new GridCacheGateway(ctx) { - @Override public void enter() { - throw new IllegalStateException("Cache has been closed: " + ctx.name()); - } - - @Override public boolean enterIfNotStopped() { - return false; - } - - @Override public boolean enterIfNotStoppedNoLock() { - return false; - } - - @Override public void leaveNoLock() { - assert false; - } - - @Override public void leave() { - assert false; - } - - @Nullable @Override public CacheOperationContext enter(@Nullable CacheOperationContext opCtx) { - throw new IllegalStateException("Cache has been closed: " + ctx.name()); - } - - @Nullable @Override public CacheOperationContext enterNoLock(@Nullable CacheOperationContext opCtx) { - throw new IllegalStateException("Cache has been closed: " + ctx.name()); - } - - @Override public void leave(CacheOperationContext prev) { - assert false; - } - - @Override public void leaveNoLock(CacheOperationContext prev) { - assert false; - } - - @Override public void stopped() { - // No-op. - } - - @Override public void onStopped() { - // No-op. - } - }; - } - - /** - * @param gate Cache gateway. - * @param opCtx Cache operation context to guard. - * @return Previous projection set on this thread. - */ - private CacheOperationContext onEnter(GridCacheGateway gate, CacheOperationContext opCtx) { - GridFutureAdapter restartFut = this.restartFut.get(); - - if (restartFut != null && !restartFut.isDone()) - throw new IgniteCacheRestartingException(new IgniteFutureImpl<>(restartFut), "Cache is restarting: " + - ctx.name()); - - return lock ? gate.enter(opCtx) : gate.enterNoLock(opCtx); - } - - /** - * @param gate Cache gateway. - * @return {@code True} if enter successful. - */ - private boolean onEnterIfNoStop(GridCacheGateway gate) { - return lock ? gate.enterIfNotStopped() : gate.enterIfNotStoppedNoLock(); - } - - /** - * @param gate Cache gateway. - * @param opCtx Operation context to guard. - */ - private void onLeave(GridCacheGateway gate, CacheOperationContext opCtx) { - if (lock) - gate.leave(opCtx); - else - gate.leaveNoLock(opCtx); - } - - /** - * @param gate Cache gateway. - */ - private void onLeave(GridCacheGateway gate) { - if (lock) - gate.leave(); - else - gate.leaveNoLock(); - } - - /** {@inheritDoc} */ - @Override public Collection lostPartitions() { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return delegate.lostPartitions(); - } - finally { - onLeave(gate, prev); - } - } - - /** {@inheritDoc} */ - @Override public void writeExternal(ObjectOutput out) throws IOException { - out.writeObject(ctx); - - out.writeObject(delegate); - - out.writeObject(opCtx); - - out.writeBoolean(lock); - } - - /** {@inheritDoc} */ - @SuppressWarnings({"unchecked"}) - @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { - ctx = (GridCacheContext)in.readObject(); - - delegate = (IgniteInternalCache)in.readObject(); - - opCtx = (CacheOperationContext)in.readObject(); - - gate = ctx.gate(); - - lock = in.readBoolean(); - } - - /** {@inheritDoc} */ - @Override public IgniteFuture rebalance() { - return new IgniteFutureImpl<>(ctx.preloader().forceRebalance()); - } - - /** {@inheritDoc} */ - @Override public IgniteFuture indexReadyFuture() { - IgniteInternalFuture fut = ctx.shared().database().indexRebuildFuture(ctx.cacheId()); - - if (fut == null) - return new IgniteFinishedFutureImpl<>(); - - return new IgniteFutureImpl<>(fut); - } + public void closeProxy(); /** - * Gets value without waiting for toplogy changes. + * Gets value without waiting for topology changes. * * @param key Key. * @return Value. */ - public V getTopologySafe(K key) { - try { - GridCacheGateway gate = this.gate; - - CacheOperationContext prev = onEnter(gate, opCtx); - - try { - return delegate.getTopologySafe(key); - } - finally { - onLeave(gate, prev); - } - } - catch (IgniteCheckedException e) { - throw cacheException(e); - } - } - - /** - * @return Restart future. - */ - public boolean isRestarting() { - return restartFut.get() != null; - } + public V getTopologySafe(K key); /** - * Restarts this cache proxy. + * @return Future that contains cache destroy operation. */ - public void restart() { - GridFutureAdapter restartFut = new GridFutureAdapter<>(); - - final GridFutureAdapter currentFut = this.restartFut.get(); - - boolean changed = this.restartFut.compareAndSet(currentFut, restartFut); - - if (changed && currentFut != null) - restartFut.listen(new IgniteInClosure>() { - @Override public void apply(IgniteInternalFuture future) { - if (future.error() != null) - currentFut.onDone(future.error()); - else - currentFut.onDone(); - } - }); - } + public IgniteFuture destroyAsync(); /** - * Mark this proxy as restarted. - * - * @param ctx New cache context. - * @param delegate New delegate. + * @return Future that contains cache close operation. */ - public void onRestarted(GridCacheContext ctx, IgniteInternalCache delegate) { - GridFutureAdapter restartFut = this.restartFut.get(); - - assert restartFut != null; - - this.ctx = ctx; - this.delegate = delegate; - - gate = ctx.gate(); - - internalProxy = new GridCacheProxyImpl<>(ctx, delegate, opCtx); - - restartFut.onDone(); - - this.restartFut.compareAndSet(restartFut, null); - } - - /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(IgniteCacheProxy.class, this); - } + public IgniteFuture closeAsync(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java new file mode 100644 index 0000000000000..b94afa1c3f862 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java @@ -0,0 +1,1810 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Lock; +import javax.cache.Cache; +import javax.cache.CacheException; +import javax.cache.configuration.CacheEntryListenerConfiguration; +import javax.cache.configuration.Configuration; +import javax.cache.expiry.ExpiryPolicy; +import javax.cache.integration.CompletionListener; +import javax.cache.processor.EntryProcessor; +import javax.cache.processor.EntryProcessorException; +import javax.cache.processor.EntryProcessorResult; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCacheRestartingException; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.cache.CacheEntry; +import org.apache.ignite.cache.CacheEntryProcessor; +import org.apache.ignite.cache.CacheManager; +import org.apache.ignite.cache.CacheMetrics; +import org.apache.ignite.cache.CachePeekMode; +import org.apache.ignite.cache.query.ContinuousQuery; +import org.apache.ignite.cache.query.FieldsQueryCursor; +import org.apache.ignite.cache.query.Query; +import org.apache.ignite.cache.query.QueryCursor; +import org.apache.ignite.cache.query.QueryDetailMetrics; +import org.apache.ignite.cache.query.QueryMetrics; +import org.apache.ignite.cache.query.ScanQuery; +import org.apache.ignite.cache.query.SpiQuery; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.cache.query.SqlQuery; +import org.apache.ignite.cache.query.TextQuery; +import org.apache.ignite.cluster.ClusterGroup; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.AsyncSupportAdapter; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.processors.cache.query.CacheQuery; +import org.apache.ignite.internal.processors.cache.query.CacheQueryFuture; +import org.apache.ignite.internal.processors.cache.query.GridCacheQueryType; +import org.apache.ignite.internal.processors.query.QueryUtils; +import org.apache.ignite.internal.util.GridCloseableIteratorAdapter; +import org.apache.ignite.internal.util.GridEmptyIterator; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl; +import org.apache.ignite.internal.util.future.IgniteFutureImpl; +import org.apache.ignite.internal.util.lang.GridCloseableIterator; +import org.apache.ignite.internal.util.lang.GridClosureException; +import org.apache.ignite.internal.util.lang.IgniteOutClosureX; +import org.apache.ignite.internal.util.tostring.GridToStringExclude; +import org.apache.ignite.internal.util.tostring.GridToStringInclude; +import org.apache.ignite.internal.util.typedef.CI1; +import org.apache.ignite.internal.util.typedef.CX1; +import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.internal.util.typedef.internal.A; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteBiPredicate; +import org.apache.ignite.lang.IgniteClosure; +import org.apache.ignite.lang.IgniteFuture; +import org.apache.ignite.lang.IgniteInClosure; +import org.apache.ignite.mxbean.CacheMetricsMXBean; +import org.apache.ignite.plugin.security.SecurityPermission; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * Cache proxy implementation. + */ +@SuppressWarnings("unchecked") +public class IgniteCacheProxyImpl extends AsyncSupportAdapter> + implements IgniteCacheProxy { + /** */ + private static final long serialVersionUID = 0L; + + /** Context. */ + private volatile GridCacheContext ctx; + + /** Delegate. */ + @GridToStringInclude + private volatile IgniteInternalCache delegate; + + /** */ + @GridToStringExclude + private CacheManager cacheMgr; + + /** Future indicates that cache is under restarting. */ + private final AtomicReference> restartFut; + + /** Flag indicates that proxy is closed. */ + private volatile boolean closed; + + /** + * Empty constructor required for {@link Externalizable}. + */ + public IgniteCacheProxyImpl() { + restartFut = new AtomicReference>(null); + } + + /** + * @param ctx Context. + * @param delegate Delegate. + * @param async Async support flag. + */ + public IgniteCacheProxyImpl( + @NotNull GridCacheContext ctx, + @NotNull IgniteInternalCache delegate, + boolean async + ) { + this(ctx, delegate, new AtomicReference>(null), async); + } + + /** + * @param ctx Context. + * @param delegate Delegate. + * @param async Async support flag. + */ + private IgniteCacheProxyImpl( + @NotNull GridCacheContext ctx, + @NotNull IgniteInternalCache delegate, + @NotNull AtomicReference> restartFut, + boolean async + ) { + super(async); + + assert ctx != null; + assert delegate != null; + + this.ctx = ctx; + this.delegate = delegate; + + this.restartFut = restartFut; + } + + /** + * @return Context. + */ + @Override + public GridCacheContext context() { + return ctx; + } + + /** {@inheritDoc} */ + @Override public IgniteCacheProxy cacheNoGate() { + return new GatewayProtectedCacheProxy<>(this, new CacheOperationContext(), false); + } + + /** {@inheritDoc} */ + @Override public CacheMetrics metrics() { + return ctx.cache().clusterMetrics(); + } + + /** {@inheritDoc} */ + @Override public CacheMetrics metrics(ClusterGroup grp) { + return ctx.cache().clusterMetrics(grp); + } + + /** {@inheritDoc} */ + @Override public CacheMetrics localMetrics() { + return ctx.cache().localMetrics(); + } + + /** {@inheritDoc} */ + @Override public CacheMetricsMXBean mxBean() { + return ctx.cache().clusterMxBean(); + } + + /** {@inheritDoc} */ + @Override public CacheMetricsMXBean localMxBean() { + return ctx.cache().localMxBean(); + } + + /** {@inheritDoc} */ + @Override public > C getConfiguration(Class clazz) { + CacheConfiguration cfg = ctx.config(); + + if (!clazz.isAssignableFrom(cfg.getClass())) + throw new IllegalArgumentException(); + + return clazz.cast(cfg); + } + + /** {@inheritDoc} */ + @Override public IgniteCache withExpiryPolicy(ExpiryPolicy plc) { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override public IgniteCache withSkipStore() { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override public IgniteCache withKeepBinary() { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override public IgniteCache withNoRetries() { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override public IgniteCache withPartitionRecover() { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override public void loadCache(@Nullable IgniteBiPredicate p, @Nullable Object... args) { + try { + if (isAsync()) { + if (ctx.cache().isLocal()) + setFuture(ctx.cache().localLoadCacheAsync(p, args)); + else + setFuture(ctx.cache().globalLoadCacheAsync(p, args)); + } + else { + if (ctx.cache().isLocal()) + ctx.cache().localLoadCache(p, args); + else + ctx.cache().globalLoadCache(p, args); + } + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture loadCacheAsync(@Nullable IgniteBiPredicate p, + @Nullable Object... args) throws CacheException { + try { + if (ctx.cache().isLocal()) + return (IgniteFuture)createFuture(ctx.cache().localLoadCacheAsync(p, args)); + else + return (IgniteFuture)createFuture(ctx.cache().globalLoadCacheAsync(p, args)); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public void localLoadCache(@Nullable IgniteBiPredicate p, @Nullable Object... args) { + try { + if (isAsync()) + setFuture(delegate.localLoadCacheAsync(p, args)); + else + delegate.localLoadCache(p, args); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture localLoadCacheAsync(@Nullable IgniteBiPredicate p, + @Nullable Object... args) throws CacheException { + return (IgniteFuture)createFuture(delegate.localLoadCacheAsync(p, args)); + } + + /** {@inheritDoc} */ + @Nullable @Override public V getAndPutIfAbsent(K key, V val) throws CacheException { + try { + if (isAsync()) { + setFuture(delegate.getAndPutIfAbsentAsync(key, val)); + + return null; + } + else + return delegate.getAndPutIfAbsent(key, val); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture getAndPutIfAbsentAsync(K key, V val) throws CacheException { + return createFuture(delegate.getAndPutIfAbsentAsync(key, val)); + } + + /** {@inheritDoc} */ + @Override public Lock lock(K key) throws CacheException { + return lockAll(Collections.singleton(key)); + } + + /** {@inheritDoc} */ + @Override public Lock lockAll(final Collection keys) { + return new CacheLockImpl<>(ctx.gate(), delegate, new CacheOperationContext(), keys); + } + + /** {@inheritDoc} */ + @Override public boolean isLocalLocked(K key, boolean byCurrThread) { + return byCurrThread ? delegate.isLockedByThread(key) : delegate.isLocked(key); + } + + /** + * @param scanQry ScanQry. + * @param transformer Transformer + * @param grp Optional cluster group. + * @return Cursor. + * @throws IgniteCheckedException If failed. + */ + @SuppressWarnings("unchecked") + private QueryCursor query( + final ScanQuery scanQry, + @Nullable final IgniteClosure transformer, + @Nullable ClusterGroup grp) + throws IgniteCheckedException { + + final CacheQuery qry; + + CacheOperationContext opCtxCall = ctx.operationContextPerCall(); + + boolean isKeepBinary = opCtxCall != null && opCtxCall.isKeepBinary(); + + IgniteBiPredicate p = scanQry.getFilter(); + + qry = ctx.queries().createScanQuery(p, transformer, scanQry.getPartition(), isKeepBinary); + + if (scanQry.getPageSize() > 0) + qry.pageSize(scanQry.getPageSize()); + + if (grp != null) + qry.projection(grp); + + final GridCloseableIterator iter = ctx.kernalContext().query().executeQuery(GridCacheQueryType.SCAN, + ctx.name(), ctx, new IgniteOutClosureX>() { + @Override public GridCloseableIterator applyx() throws IgniteCheckedException { + final GridCloseableIterator iter0 = qry.executeScanQuery(); + + final boolean needToConvert = transformer == null; + + return new GridCloseableIteratorAdapter() { + @Override protected R onNext() throws IgniteCheckedException { + Object next = iter0.nextX(); + + if (needToConvert) { + Map.Entry entry = (Map.Entry)next; + + return (R)new CacheEntryImpl<>(entry.getKey(), entry.getValue()); + } + + return (R)next; + } + + @Override protected boolean onHasNext() throws IgniteCheckedException { + return iter0.hasNextX(); + } + + @Override protected void onClose() throws IgniteCheckedException { + iter0.close(); + } + }; + } + }, true); + + return new QueryCursorImpl<>(iter); + } + + /** + * @param filter Filter. + * @param grp Optional cluster group. + * @return Cursor. + * @throws IgniteCheckedException If failed. + */ + @SuppressWarnings("unchecked") + private QueryCursor> query(final Query filter, @Nullable ClusterGroup grp) + throws IgniteCheckedException { + final CacheQuery qry; + + CacheOperationContext opCtxCall = ctx.operationContextPerCall(); + + boolean isKeepBinary = opCtxCall != null && opCtxCall.isKeepBinary(); + + final CacheQueryFuture fut; + + if (filter instanceof TextQuery) { + TextQuery p = (TextQuery)filter; + + qry = ctx.queries().createFullTextQuery(p.getType(), p.getText(), isKeepBinary); + + if (grp != null) + qry.projection(grp); + + fut = ctx.kernalContext().query().executeQuery(GridCacheQueryType.TEXT, p.getText(), ctx, + new IgniteOutClosureX>>() { + @Override public CacheQueryFuture> applyx() { + return qry.execute(); + } + }, false); + } + else if (filter instanceof SpiQuery) { + qry = ctx.queries().createSpiQuery(isKeepBinary); + + if (grp != null) + qry.projection(grp); + + fut = ctx.kernalContext().query().executeQuery(GridCacheQueryType.SPI, filter.getClass().getSimpleName(), + ctx, new IgniteOutClosureX>>() { + @Override public CacheQueryFuture> applyx() { + return qry.execute(((SpiQuery)filter).getArgs()); + } + }, false); + } + else { + if (filter instanceof SqlFieldsQuery) + throw new CacheException("Use methods 'queryFields' and 'localQueryFields' for " + + SqlFieldsQuery.class.getSimpleName() + "."); + + throw new CacheException("Unsupported query type: " + filter); + } + + return new QueryCursorImpl<>(new GridCloseableIteratorAdapter>() { + /** */ + private Cache.Entry cur; + + @Override protected Entry onNext() throws IgniteCheckedException { + if (!onHasNext()) + throw new NoSuchElementException(); + + Cache.Entry e = cur; + + cur = null; + + return e; + } + + @Override protected boolean onHasNext() throws IgniteCheckedException { + if (cur != null) + return true; + + Object next = fut.next(); + + // Workaround a bug: if IndexingSpi is configured future represents Iterator + // instead of Iterator due to IndexingSpi interface. + if (next == null) + return false; + + if (next instanceof Cache.Entry) + cur = (Cache.Entry)next; + else { + Map.Entry e = (Map.Entry)next; + + cur = new CacheEntryImpl(e.getKey(), e.getValue()); + } + + return true; + } + + @Override protected void onClose() throws IgniteCheckedException { + fut.cancel(); + } + }); + } + + /** + * @param loc Enforce local. + * @return Local node cluster group. + */ + private ClusterGroup projection(boolean loc) { + if (loc || ctx.isLocal() || ctx.isReplicatedAffinityNode()) + return ctx.kernalContext().grid().cluster().forLocal(); + + if (ctx.isReplicated()) + return ctx.kernalContext().grid().cluster().forDataNodes(ctx.name()).forRandom(); + + return null; + } + + /** + * Executes continuous query. + * + * @param qry Query. + * @param loc Local flag. + * @param keepBinary Keep binary flag. + * @return Initial iteration cursor. + */ + @SuppressWarnings("unchecked") + private QueryCursor> queryContinuous(ContinuousQuery qry, boolean loc, boolean keepBinary) { + if (qry.getInitialQuery() instanceof ContinuousQuery) + throw new IgniteException("Initial predicate for continuous query can't be an instance of another " + + "continuous query. Use SCAN or SQL query for initial iteration."); + + if (qry.getLocalListener() == null) + throw new IgniteException("Mandatory local listener is not set for the query: " + qry); + + if (qry.getRemoteFilter() != null && qry.getRemoteFilterFactory() != null) + throw new IgniteException("Should be used either RemoterFilter or RemoteFilterFactory."); + + try { + final UUID routineId = ctx.continuousQueries().executeQuery( + qry.getLocalListener(), + qry.getRemoteFilter(), + qry.getRemoteFilterFactory(), + qry.getPageSize(), + qry.getTimeInterval(), + qry.isAutoUnsubscribe(), + loc, + keepBinary, + qry.isIncludeExpired()); + + final QueryCursor> cur = + qry.getInitialQuery() != null ? query(qry.getInitialQuery()) : null; + + return new QueryCursor>() { + @Override public Iterator> iterator() { + return cur != null ? cur.iterator() : new GridEmptyIterator>(); + } + + @Override public List> getAll() { + return cur != null ? cur.getAll() : Collections.>emptyList(); + } + + @Override public void close() { + if (cur != null) + cur.close(); + + try { + ctx.kernalContext().continuous().stopRoutine(routineId).get(); + } + catch (IgniteCheckedException e) { + throw U.convertException(e); + } + } + }; + } + catch (IgniteCheckedException e) { + throw U.convertException(e); + } + } + + /** {@inheritDoc} */ + @SuppressWarnings("unchecked") + @Override public FieldsQueryCursor> query(SqlFieldsQuery qry) { + return (FieldsQueryCursor>)query((Query)qry); + } + + /** {@inheritDoc} */ + @SuppressWarnings("unchecked") + @Override public QueryCursor query(Query qry) { + A.notNull(qry, "qry"); + try { + ctx.checkSecurity(SecurityPermission.CACHE_READ); + + validate(qry); + + convertToBinary(qry); + + CacheOperationContext opCtxCall = ctx.operationContextPerCall(); + + boolean keepBinary = opCtxCall != null && opCtxCall.isKeepBinary(); + + if (qry instanceof ContinuousQuery) + return (QueryCursor)queryContinuous((ContinuousQuery)qry, qry.isLocal(), keepBinary); + + if (qry instanceof SqlQuery) + return (QueryCursor)ctx.kernalContext().query().querySql(ctx, (SqlQuery)qry, keepBinary); + + if (qry instanceof SqlFieldsQuery) + return (FieldsQueryCursor)ctx.kernalContext().query().querySqlFields(ctx, (SqlFieldsQuery)qry, + keepBinary); + + if (qry instanceof ScanQuery) + return query((ScanQuery)qry, null, projection(qry.isLocal())); + + return (QueryCursor)query(qry, projection(qry.isLocal())); + } + catch (Exception e) { + if (e instanceof CacheException) + throw (CacheException)e; + + throw new CacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public QueryCursor query(Query qry, IgniteClosure transformer) { + A.notNull(qry, "qry"); + A.notNull(transformer, "transformer"); + + if (!(qry instanceof ScanQuery)) + throw new UnsupportedOperationException("Transformers are supported only for SCAN queries."); + + try { + ctx.checkSecurity(SecurityPermission.CACHE_READ); + + validate(qry); + + return query((ScanQuery)qry, transformer, projection(qry.isLocal())); + } + catch (Exception e) { + if (e instanceof CacheException) + throw (CacheException)e; + + throw new CacheException(e); + } + } + + /** + * Convert query arguments to BinaryObjects if binary marshaller used. + * + * @param qry Query. + */ + private void convertToBinary(final Query qry) { + if (ctx.binaryMarshaller()) { + if (qry instanceof SqlQuery) { + final SqlQuery sqlQry = (SqlQuery) qry; + + convertToBinary(sqlQry.getArgs()); + } + else if (qry instanceof SpiQuery) { + final SpiQuery spiQry = (SpiQuery) qry; + + convertToBinary(spiQry.getArgs()); + } + else if (qry instanceof SqlFieldsQuery) { + final SqlFieldsQuery fieldsQry = (SqlFieldsQuery) qry; + + convertToBinary(fieldsQry.getArgs()); + } + } + } + + /** + * Converts query arguments to BinaryObjects if binary marshaller used. + * + * @param args Arguments. + */ + private void convertToBinary(final Object[] args) { + if (args == null) + return; + + for (int i = 0; i < args.length; i++) + args[i] = ctx.cacheObjects().binary().toBinary(args[i]); + } + + /** + * Checks query. + * + * @param qry Query + * @throws CacheException If query indexing disabled for sql query. + */ + private void validate(Query qry) { + if (!QueryUtils.isEnabled(ctx.config()) && !(qry instanceof ScanQuery) && + !(qry instanceof ContinuousQuery) && !(qry instanceof SpiQuery)) + throw new CacheException("Indexing is disabled for cache: " + ctx.cache().name() + + ". Use setIndexedTypes or setTypeMetadata methods on CacheConfiguration to enable."); + + if (!ctx.kernalContext().query().moduleEnabled() && + (qry instanceof SqlQuery || qry instanceof SqlFieldsQuery || qry instanceof TextQuery)) + throw new CacheException("Failed to execute query. Add module 'ignite-indexing' to the classpath " + + "of all Ignite nodes."); + } + + /** {@inheritDoc} */ + @Override public Iterable> localEntries(CachePeekMode... peekModes) throws CacheException { + try { + return delegate.localEntries(peekModes); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public QueryMetrics queryMetrics() { + return delegate.context().queries().metrics(); + } + + /** {@inheritDoc} */ + @Override public void resetQueryMetrics() { + delegate.context().queries().resetMetrics(); + } + + /** {@inheritDoc} */ + @Override public Collection queryDetailMetrics() { + return delegate.context().queries().detailMetrics(); + } + + /** {@inheritDoc} */ + @Override public void resetQueryDetailMetrics() { + delegate.context().queries().resetDetailMetrics(); + } + + /** {@inheritDoc} */ + @Override public void localEvict(Collection keys) { + delegate.evictAll(keys); + } + + /** {@inheritDoc} */ + @Nullable @Override public V localPeek(K key, CachePeekMode... peekModes) { + try { + return delegate.localPeek(key, peekModes, null); + } + catch (IgniteException | IgniteCheckedException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public int size(CachePeekMode... peekModes) throws CacheException { + try { + if (isAsync()) { + setFuture(delegate.sizeAsync(peekModes)); + + return 0; + } + else + return delegate.size(peekModes); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture sizeAsync(CachePeekMode... peekModes) throws CacheException { + return createFuture(delegate.sizeAsync(peekModes)); + } + + /** {@inheritDoc} */ + @Override public long sizeLong(CachePeekMode... peekModes) throws CacheException { + try { + if (isAsync()) { + setFuture(delegate.sizeLongAsync(peekModes)); + + return 0; + } + else + return delegate.sizeLong(peekModes); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture sizeLongAsync(CachePeekMode... peekModes) throws CacheException { + return createFuture(delegate.sizeLongAsync(peekModes)); + } + + /** {@inheritDoc} */ + @Override public long sizeLong(int part, CachePeekMode... peekModes) throws CacheException { + try { + if (isAsync()) { + setFuture(delegate.sizeLongAsync(part, peekModes)); + + return 0; + } + else + return delegate.sizeLong(part, peekModes); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture sizeLongAsync(int part, CachePeekMode... peekModes) throws CacheException { + return createFuture(delegate.sizeLongAsync(part, peekModes)); + } + + /** {@inheritDoc} */ + @Override public int localSize(CachePeekMode... peekModes) { + try { + return delegate.localSize(peekModes); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public long localSizeLong(CachePeekMode... peekModes) { + try { + return delegate.localSizeLong(peekModes); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public long localSizeLong(int part, CachePeekMode... peekModes) { + try { + return delegate.localSizeLong(part, peekModes); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public V get(K key) { + try { + if (isAsync()) { + setFuture(delegate.getAsync(key)); + + return null; + } + else + return delegate.get(key); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture getAsync(K key) { + return createFuture(delegate.getAsync(key)); + } + + /** {@inheritDoc} */ + @Override public CacheEntry getEntry(K key) { + try { + if (isAsync()) { + setFuture(delegate.getEntryAsync(key)); + + return null; + } + else + return delegate.getEntry(key); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture> getEntryAsync(K key) { + return createFuture(delegate.getEntryAsync(key)); + } + + /** {@inheritDoc} */ + @Override public Map getAll(Set keys) { + try { + if (isAsync()) { + setFuture(delegate.getAllAsync(keys)); + + return null; + } + else + return delegate.getAll(keys); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture> getAllAsync(Set keys) { + return createFuture(delegate.getAllAsync(keys)); + } + + /** {@inheritDoc} */ + @Override public Collection> getEntries(Set keys) { + try { + if (isAsync()) { + setFuture(delegate.getEntriesAsync(keys)); + + return null; + } + else + return delegate.getEntries(keys); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture>> getEntriesAsync(Set keys) { + return createFuture(delegate.getEntriesAsync(keys)); + } + + /** {@inheritDoc} */ + @Override public Map getAllOutTx(Set keys) { + try { + if (isAsync()) { + setFuture(delegate.getAllOutTxAsync(keys)); + + return null; + } + else + return delegate.getAllOutTx(keys); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture> getAllOutTxAsync(Set keys) { + return createFuture(delegate.getAllOutTxAsync(keys)); + } + + /** + * @param keys Keys. + * @return Values map. + */ + public Map getAll(Collection keys) { + try { + if (isAsync()) { + setFuture(delegate.getAllAsync(keys)); + + return null; + } + else + return delegate.getAll(keys); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public boolean containsKey(K key) { + if (isAsync()) { + setFuture(delegate.containsKeyAsync(key)); + + return false; + } + else + return delegate.containsKey(key); + } + + /** {@inheritDoc} */ + @Override public IgniteFuture containsKeyAsync(K key) { + return createFuture(delegate.containsKeyAsync(key)); + } + + /** {@inheritDoc} */ + @Override public boolean containsKeys(Set keys) { + if (isAsync()) { + setFuture(delegate.containsKeysAsync(keys)); + + return false; + } + else + return delegate.containsKeys(keys); + } + + /** {@inheritDoc} */ + @Override public IgniteFuture containsKeysAsync(Set keys) { + return createFuture(delegate.containsKeysAsync(keys)); + } + + /** {@inheritDoc} */ + @Override public void loadAll( + Set keys, + boolean replaceExisting, + @Nullable final CompletionListener completionLsnr + ) { + IgniteInternalFuture fut = ctx.cache().loadAll(keys, replaceExisting); + + if (completionLsnr != null) { + fut.listen(new CI1>() { + @Override public void apply(IgniteInternalFuture fut) { + try { + fut.get(); + + completionLsnr.onCompletion(); + } + catch (IgniteCheckedException e) { + completionLsnr.onException(cacheException(e)); + } + } + }); + } + } + + /** {@inheritDoc} */ + @Override public void put(K key, V val) { + try { + if (isAsync()) + setFuture(putAsync0(key, val)); + else + delegate.put(key, val); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture putAsync(K key, V val) { + return createFuture(putAsync0(key, val)); + } + + /** + * Put async internal operation implementation. + * + * @param key Key. + * @param val Value. + * @return Internal future. + */ + private IgniteInternalFuture putAsync0(K key, V val) { + IgniteInternalFuture fut = delegate.putAsync(key, val); + + return fut.chain(new CX1, Void>() { + @Override public Void applyx(IgniteInternalFuture fut1) throws IgniteCheckedException { + try { + fut1.get(); + } + catch (RuntimeException e) { + throw new GridClosureException(e); + } + + return null; + } + }); + } + + /** {@inheritDoc} */ + @Override public V getAndPut(K key, V val) { + try { + if (isAsync()) { + setFuture(delegate.getAndPutAsync(key, val)); + + return null; + } + else + return delegate.getAndPut(key, val); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture getAndPutAsync(K key, V val) { + return createFuture(delegate.getAndPutAsync(key, val)); + } + + /** {@inheritDoc} */ + @Override public void putAll(Map map) { + try { + if (isAsync()) + setFuture(delegate.putAllAsync(map)); + else + delegate.putAll(map); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture putAllAsync(Map map) { + return (IgniteFuture)createFuture(delegate.putAllAsync(map)); + } + + /** {@inheritDoc} */ + @Override public boolean putIfAbsent(K key, V val) { + try { + if (isAsync()) { + setFuture(delegate.putIfAbsentAsync(key, val)); + + return false; + } + else + return delegate.putIfAbsent(key, val); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture putIfAbsentAsync(K key, V val) { + return createFuture(delegate.putIfAbsentAsync(key, val)); + } + + /** {@inheritDoc} */ + @Override public boolean remove(K key) { + try { + if (isAsync()) { + setFuture(delegate.removeAsync(key)); + + return false; + } + else + return delegate.remove(key); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture removeAsync(K key) { + return createFuture(delegate.removeAsync(key)); + } + + /** {@inheritDoc} */ + @Override public boolean remove(K key, V oldVal) { + try { + if (isAsync()) { + setFuture(delegate.removeAsync(key, oldVal)); + + return false; + } + else + return delegate.remove(key, oldVal); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture removeAsync(K key, V oldVal) { + return createFuture(delegate.removeAsync(key, oldVal)); + } + + /** {@inheritDoc} */ + @Override public V getAndRemove(K key) { + try { + if (isAsync()) { + setFuture(delegate.getAndRemoveAsync(key)); + + return null; + } + else + return delegate.getAndRemove(key); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture getAndRemoveAsync(K key) { + return createFuture(delegate.getAndRemoveAsync(key)); + } + + /** {@inheritDoc} */ + @Override public boolean replace(K key, V oldVal, V newVal) { + try { + if (isAsync()) { + setFuture(delegate.replaceAsync(key, oldVal, newVal)); + + return false; + } + else + return delegate.replace(key, oldVal, newVal); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture replaceAsync(K key, V oldVal, V newVal) { + return createFuture(delegate.replaceAsync(key, oldVal, newVal)); + } + + /** {@inheritDoc} */ + @Override public boolean replace(K key, V val) { + try { + if (isAsync()) { + setFuture(delegate.replaceAsync(key, val)); + + return false; + } + else + return delegate.replace(key, val); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture replaceAsync(K key, V val) { + return createFuture(delegate.replaceAsync(key, val)); + } + + /** {@inheritDoc} */ + @Override public V getAndReplace(K key, V val) { + try { + if (isAsync()) { + setFuture(delegate.getAndReplaceAsync(key, val)); + + return null; + } + else + return delegate.getAndReplace(key, val); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture getAndReplaceAsync(K key, V val) { + return createFuture(delegate.getAndReplaceAsync(key, val)); + } + + /** {@inheritDoc} */ + @Override public void removeAll(Set keys) { + try { + if (isAsync()) + setFuture(delegate.removeAllAsync(keys)); + else + delegate.removeAll(keys); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture removeAllAsync(Set keys) { + return (IgniteFuture)createFuture(delegate.removeAllAsync(keys)); + } + + /** {@inheritDoc} */ + @Override public void removeAll() { + try { + if (isAsync()) + setFuture(delegate.removeAllAsync()); + else + delegate.removeAll(); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture removeAllAsync() { + return (IgniteFuture)createFuture(delegate.removeAllAsync()); + } + + /** {@inheritDoc} */ + @Override public void clear(K key) { + try { + if (isAsync()) + setFuture(delegate.clearAsync(key)); + else + delegate.clear(key); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture clearAsync(K key) { + return (IgniteFuture)createFuture(delegate.clearAsync(key)); + } + + /** {@inheritDoc} */ + @Override public void clearAll(Set keys) { + try { + if (isAsync()) + setFuture(delegate.clearAllAsync(keys)); + else + delegate.clearAll(keys); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture clearAllAsync(Set keys) { + return (IgniteFuture)createFuture(delegate.clearAllAsync(keys)); + } + + /** {@inheritDoc} */ + @Override public void clear() { + try { + if (isAsync()) + setFuture(delegate.clearAsync()); + else + delegate.clear(); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture clearAsync() { + return (IgniteFuture)createFuture(delegate.clearAsync()); + } + + /** {@inheritDoc} */ + @Override public void localClear(K key) { + delegate.clearLocally(key); + } + + /** {@inheritDoc} */ + @Override public void localClearAll(Set keys) { + for (K key : keys) + delegate.clearLocally(key); + } + + /** {@inheritDoc} */ + @Override public T invoke(K key, EntryProcessor entryProcessor, Object... args) + throws EntryProcessorException { + try { + if (isAsync()) { + setFuture(invokeAsync0(key, entryProcessor, args)); + + return null; + } + else { + EntryProcessorResult res = delegate.invoke(key, entryProcessor, args); + + return res != null ? res.get() : null; + } + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture invokeAsync(K key, EntryProcessor entryProcessor, + Object... args) { + return createFuture(invokeAsync0(key, entryProcessor, args)); + } + + /** + * Invoke async operation internal implementation. + * + * @param key Key. + * @param entryProcessor Processor. + * @param args Arguments. + * @return Internal future. + */ + private IgniteInternalFuture invokeAsync0(K key, EntryProcessor entryProcessor, Object[] args) { + IgniteInternalFuture> fut = delegate.invokeAsync(key, entryProcessor, args); + + return fut.chain(new CX1>, T>() { + @Override public T applyx(IgniteInternalFuture> fut1) + throws IgniteCheckedException { + try { + EntryProcessorResult res = fut1.get(); + + return res != null ? res.get() : null; + } + catch (RuntimeException e) { + throw new GridClosureException(e); + } + } + }); + } + + + /** {@inheritDoc} */ + @Override public T invoke(K key, CacheEntryProcessor entryProcessor, Object... args) + throws EntryProcessorException { + return invoke(key, (EntryProcessor)entryProcessor, args); + } + + /** {@inheritDoc} */ + @Override public IgniteFuture invokeAsync(K key, CacheEntryProcessor entryProcessor, + Object... args) { + return invokeAsync(key, (EntryProcessor)entryProcessor, args); + } + + /** + * @param topVer Locked topology version. + * @param key Key. + * @param entryProcessor Entry processor. + * @param args Arguments. + * @return Invoke result. + */ + public T invoke(@Nullable AffinityTopologyVersion topVer, + K key, + EntryProcessor entryProcessor, + Object... args) { + try { + if (isAsync()) + throw new UnsupportedOperationException(); + else { + EntryProcessorResult res = delegate.invoke(topVer, key, entryProcessor, args); + + return res != null ? res.get() : null; + } + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public Map> invokeAll(Set keys, + EntryProcessor entryProcessor, + Object... args) { + try { + if (isAsync()) { + setFuture(delegate.invokeAllAsync(keys, entryProcessor, args)); + + return null; + } + else + return delegate.invokeAll(keys, entryProcessor, args); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture>> invokeAllAsync(Set keys, + EntryProcessor entryProcessor, Object... args) { + return createFuture(delegate.invokeAllAsync(keys, entryProcessor, args)); + } + + /** {@inheritDoc} */ + @Override public Map> invokeAll(Set keys, + CacheEntryProcessor entryProcessor, + Object... args) { + try { + if (isAsync()) { + setFuture(delegate.invokeAllAsync(keys, entryProcessor, args)); + + return null; + } + else + return delegate.invokeAll(keys, entryProcessor, args); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture>> invokeAllAsync(Set keys, + CacheEntryProcessor entryProcessor, Object... args) { + return createFuture(delegate.invokeAllAsync(keys, entryProcessor, args)); + } + + /** {@inheritDoc} */ + @Override public Map> invokeAll( + Map> map, + Object... args) { + try { + if (isAsync()) { + setFuture(delegate.invokeAllAsync(map, args)); + + return null; + } + else + return delegate.invokeAll(map, args); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public IgniteFuture>> invokeAllAsync( + Map> map, Object... args) { + return createFuture(delegate.invokeAllAsync(map, args)); + } + + /** {@inheritDoc} */ + @Override public String getName() { + return delegate.name(); + } + + /** {@inheritDoc} */ + @Override public CacheManager getCacheManager() { + return cacheMgr; + } + + /** + * @param cacheMgr Cache manager. + */ + public void setCacheManager(CacheManager cacheMgr) { + this.cacheMgr = cacheMgr; + } + + /** {@inheritDoc} */ + @Override public void destroy() { + destroyAsync().get(); + } + + /** {@inheritDoc} */ + @Override public IgniteFuture destroyAsync() { + return new IgniteFutureImpl<>(ctx.kernalContext().cache().dynamicDestroyCache(ctx.name(), false, true, false)); + } + + /** {@inheritDoc} */ + @Override public void close() { + closeAsync().get(); + } + + /** {@inheritDoc} */ + @Override public IgniteFuture closeAsync() { + return new IgniteFutureImpl<>(ctx.kernalContext().cache().dynamicCloseCache(ctx.name())); + } + + /** {@inheritDoc} */ + @Override public boolean isClosed() { + return ctx.kernalContext().cache().context().closed(ctx); + } + + /** {@inheritDoc} */ + @SuppressWarnings("unchecked") + @Override public T unwrap(Class clazz) { + if (clazz.isAssignableFrom(getClass())) + return (T)this; + else if (clazz.isAssignableFrom(IgniteEx.class)) + return (T)ctx.grid(); + + throw new IllegalArgumentException("Unwrapping to class is not supported: " + clazz); + } + + /** {@inheritDoc} */ + @Override public void registerCacheEntryListener(CacheEntryListenerConfiguration lsnrCfg) { + try { + CacheOperationContext opCtx = ctx.operationContextPerCall(); + + ctx.continuousQueries().executeJCacheQuery(lsnrCfg, false, opCtx != null && opCtx.isKeepBinary()); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public void deregisterCacheEntryListener(CacheEntryListenerConfiguration lsnrCfg) { + try { + ctx.continuousQueries().cancelJCacheQuery(lsnrCfg); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override public Iterator> iterator() { + try { + return ctx.cache().igniteIterator(); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** {@inheritDoc} */ + @Override protected IgniteCache createAsyncInstance() { + return new IgniteCacheProxyImpl( + ctx, + delegate, + true + ); + } + + /** + * Creates projection that will operate with binary objects.

Projection returned by this method will force + * cache not to deserialize binary objects, so keys and values will be returned from cache API methods without + * changes. Therefore, signature of the projection can contain only following types:

  • {@code BinaryObject} + * for binary classes
  • All primitives (byte, int, ...) and there boxed versions (Byte, Integer, ...)
    • + *
  • Arrays of primitives (byte[], int[], ...)
  • {@link String} and array of {@link String}s
    • + *
  • {@link UUID} and array of {@link UUID}s
  • {@link Date} and array of {@link Date}s
  • {@link + * java.sql.Timestamp} and array of {@link java.sql.Timestamp}s
  • Enums and array of enums
  • Maps, + * collections and array of objects (but objects inside them will still be converted if they are binary)
    • + *

For example, if you use {@link Integer} as a key and {@code Value} class as a value (which will be + * stored in binary format), you should acquire following projection to avoid deserialization: + * 

+     * IgniteInternalCache prj = cache.keepBinary();
+     *
+     * // Value is not deserialized and returned in binary format.
+     * GridBinaryObject po = prj.get(1);
+     *
+ *

Note that this method makes sense only if cache is working in binary mode ({@code + * CacheConfiguration#isBinaryEnabled()} returns {@code true}. If not, this method is no-op and will return + * current projection. + * + * @return Projection for binary objects. + */ + @Override + @SuppressWarnings("unchecked") + public IgniteCache keepBinary() { + throw new UnsupportedOperationException(); + } + + /** + * @param dataCenterId Data center ID. + * @return Projection for data center id. + */ + @Override + @SuppressWarnings("unchecked") + public IgniteCache withDataCenterId(byte dataCenterId) { + throw new UnsupportedOperationException(); + } + + /** + * @return Cache with skip store enabled. + */ + @Override + public IgniteCache skipStore() { + throw new UnsupportedOperationException(); + } + + /** + * Method converts exception to IgniteCacheRestartingException in case of cache restarting + * or to CacheException in other cases. + * + * @param e {@code IgniteCheckedException} or {@code IgniteException}. + * @return Cache exception. + */ + private RuntimeException cacheException(Exception e) { + GridFutureAdapter restartFut = this.restartFut.get(); + + if (restartFut != null && !restartFut.isDone()) { + if (X.hasCause(e, CacheStoppedException.class) || X.hasSuppressed(e, CacheStoppedException.class)) + throw new IgniteCacheRestartingException(new IgniteFutureImpl<>(restartFut), "Cache is restarting: " + + ctx.name()); + } + + if (e instanceof IgniteCheckedException) + return CU.convertToCacheException((IgniteCheckedException) e); + + if (e instanceof RuntimeException) + return (RuntimeException) e; + + throw new IllegalStateException("Unknown exception", e); + } + + /** + * @param fut Future for async operation. + */ + private void setFuture(IgniteInternalFuture fut) { + curFut.set(createFuture(fut)); + } + + /** {@inheritDoc} */ + @Override protected IgniteFuture createFuture(IgniteInternalFuture fut) { + return new IgniteCacheFutureImpl<>(fut); + } + + /** + * @return Internal proxy. + */ + @Override + public GridCacheProxyImpl internalProxy() { + return new GridCacheProxyImpl<>(ctx, delegate, ctx.operationContextPerCall()); + } + + /** + * @return {@code True} if proxy was closed. + */ + @Override public boolean isProxyClosed() { + return closed; + } + + /** + * Closes this proxy instance. + */ + @Override public void closeProxy() { + closed = true; + } + + /** {@inheritDoc} */ + @Override public Collection lostPartitions() { + return delegate.lostPartitions(); + } + + /** {@inheritDoc} */ + @Override public void writeExternal(ObjectOutput out) throws IOException { + out.writeObject(ctx); + + out.writeObject(delegate); + } + + /** {@inheritDoc} */ + @SuppressWarnings({"unchecked"}) + @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + ctx = (GridCacheContext)in.readObject(); + + delegate = (IgniteInternalCache)in.readObject(); + } + + /** {@inheritDoc} */ + @Override public IgniteFuture rebalance() { + return new IgniteFutureImpl<>(ctx.preloader().forceRebalance()); + } + + /** {@inheritDoc} */ + @Override public IgniteFuture indexReadyFuture() { + IgniteInternalFuture fut = ctx.shared().database().indexRebuildFuture(ctx.cacheId()); + + if (fut == null) + return new IgniteFinishedFutureImpl<>(); + + return new IgniteFutureImpl<>(fut); + } + + /** + * Gets value without waiting for toplogy changes. + * + * @param key Key. + * @return Value. + */ + @Override + public V getTopologySafe(K key) { + try { + return delegate.getTopologySafe(key); + } + catch (IgniteCheckedException | IgniteException e) { + throw cacheException(e); + } + } + + /** + * Throws {@code IgniteCacheRestartingException} if proxy is restarting. + */ + public void checkRestart() { + if (isRestarting()) + throw new IgniteCacheRestartingException(new IgniteFutureImpl<>(restartFut.get()), "Cache is restarting: " + + context().name()); + } + + /** + * @return True if proxy is restarting, false in other case. + */ + public boolean isRestarting() { + return restartFut != null && restartFut.get() != null; + } + + /** + * Restarts this cache proxy. + */ + public void restart() { + GridFutureAdapter restartFut = new GridFutureAdapter<>(); + + final GridFutureAdapter currentFut = this.restartFut.get(); + + boolean changed = this.restartFut.compareAndSet(currentFut, restartFut); + + if (changed && currentFut != null) + restartFut.listen(new IgniteInClosure>() { + @Override public void apply(IgniteInternalFuture future) { + if (future.error() != null) + currentFut.onDone(future.error()); + else + currentFut.onDone(); + } + }); + } + + /** + * Mark this proxy as restarted. + * + * @param ctx New cache context. + * @param delegate New delegate. + */ + public void onRestarted(GridCacheContext ctx, IgniteInternalCache delegate) { + GridFutureAdapter restartFut = this.restartFut.get(); + + assert restartFut != null; + + this.ctx = ctx; + this.delegate = delegate; + + restartFut.onDone(); + + this.restartFut.compareAndSet(restartFut, null); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(IgniteCacheProxyImpl.class, this); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/dr/IgniteDrDataStreamerCacheUpdater.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/dr/IgniteDrDataStreamerCacheUpdater.java index 0cec1fe1cbc6e..61ab122349b76 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/dr/IgniteDrDataStreamerCacheUpdater.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/dr/IgniteDrDataStreamerCacheUpdater.java @@ -62,7 +62,7 @@ public class IgniteDrDataStreamerCacheUpdater implements StreamReceiver loadCacheAsync0(BinaryRawReaderEx reader, boolean loc } case OP_WITH_NO_RETRIES: { - CacheOperationContext opCtx = cache.operationContext(); + CacheOperationContext opCtx = cache.context().operationContextPerCall(); if (opCtx != null && opCtx.noRetries()) return this; @@ -1018,7 +1017,9 @@ private IgniteFuture loadCacheAsync0(BinaryRawReaderEx reader, boolean loc } case OP_WITH_SKIP_STORE: { - if (cache.delegate().skipStore()) + CacheOperationContext opCtx = cache.context().operationContextPerCall(); + + if (opCtx != null && opCtx.skipStore()) return this; return copy(rawCache.withSkipStore(), keepBinary); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheEntryProcessorCopySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheEntryProcessorCopySelfTest.java index 7affa8ccdadeb..7005e146fd8ab 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheEntryProcessorCopySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheEntryProcessorCopySelfTest.java @@ -31,6 +31,7 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.binary.BinaryMarshaller; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; @@ -149,7 +150,15 @@ private void doTest(boolean cpOnRead, final boolean mutate, int expVal, int expC } }); - CacheObject obj = ((GridCacheAdapter)((IgniteCacheProxy)cache).delegate()).peekEx(0).peekVisibleValue(); + GridCacheAdapter ca = (GridCacheAdapter)((IgniteCacheProxy)cache).internalProxy().delegate(); + + GridCacheEntryEx entry = ca.entryEx(0); + + entry.unswap(); + + CacheObject obj = entry.peekVisibleValue(); + + ca.context().evicts().touch(entry, AffinityTopologyVersion.NONE); int actCnt = cnt.get(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheStopAndDestroySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheStopAndDestroySelfTest.java index f67e2472461fe..c53bc4bb0767c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheStopAndDestroySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheStopAndDestroySelfTest.java @@ -768,6 +768,7 @@ public void testTckStyleCreateDestroyClose() throws Exception { cache.close(); + // Check second close succeeds without exception. cache.close(); try { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java index 822537c580160..9376971a8a7ae 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java @@ -34,6 +34,7 @@ import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.CachePeekMode; import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.cache.store.CacheStore; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; @@ -246,6 +247,7 @@ protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throw } } + cfg.setAffinity(new RendezvousAffinityFunction(false, 4096)); cfg.setCacheMode(cacheMode()); cfg.setAtomicityMode(atomicityMode()); cfg.setWriteSynchronizationMode(writeSynchronization()); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheOnCopyFlagAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheOnCopyFlagAbstractSelfTest.java index 80404ce8f5dbb..2a90bf608b3f1 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheOnCopyFlagAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheOnCopyFlagAbstractSelfTest.java @@ -178,7 +178,7 @@ private void interceptor() throws Exception { cache.put(key, val); CacheObject obj = - ((GridCacheAdapter)((IgniteCacheProxy)cache).delegate()).peekEx(key).peekVisibleValue(); + ((GridCacheAdapter)((IgniteCacheProxy)cache).internalProxy().delegate()).peekEx(key).peekVisibleValue(); // Check thar internal entry wasn't changed. assertEquals(i, getValue(obj, cache)); @@ -211,7 +211,7 @@ private void interceptor() throws Exception { cache.put(key, newTestVal); - obj = ((GridCacheAdapter)((IgniteCacheProxy)cache).delegate()).peekEx(key).peekVisibleValue(); + obj = ((GridCacheAdapter)((IgniteCacheProxy)cache).internalProxy().delegate()).peekEx(key).peekVisibleValue(); // Check thar internal entry wasn't changed. assertEquals(-i, getValue(obj, cache)); @@ -290,7 +290,7 @@ private void invokeAndInterceptor() throws Exception { }); CacheObject obj = - ((GridCacheAdapter)((IgniteCacheProxy)cache).delegate()).peekEx(key).peekVisibleValue(); + ((GridCacheAdapter)((IgniteCacheProxy)cache).internalProxy().delegate()).peekEx(key).peekVisibleValue(); assertNotEquals(WRONG_VALUE, getValue(obj, cache)); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheValueConsistencyAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheValueConsistencyAbstractSelfTest.java index 3c5fe0e512fa6..e068252c712d4 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheValueConsistencyAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheValueConsistencyAbstractSelfTest.java @@ -124,7 +124,7 @@ public void testPutRemove() throws Exception { info("Node is reported as NOT affinity node for key [key=" + key + ", nodeId=" + locNode.id() + ']'); - if (nearEnabled() && cache == cache0) + if (nearEnabled() && cache.equals(cache0)) assertEquals((Integer)i, cache0.localPeek(key)); else assertNull(cache0.localPeek(key)); @@ -184,7 +184,7 @@ public void testPutRemoveAll() throws Exception { info("Node is reported as NOT affinity node for key [key=" + key + ", nodeId=" + locNode.id() + ']'); - if (nearEnabled() && cache == cache0) + if (nearEnabled() && cache.equals(cache0)) assertEquals((Integer)i, cache0.localPeek(key)); else assertNull(cache0.localPeek(key)); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java index dcba92f3d3fc6..8d5462dcf4bd5 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java @@ -17,8 +17,6 @@ package org.apache.ignite.internal.processors.cache; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; @@ -49,12 +47,13 @@ import javax.cache.processor.EntryProcessorException; import javax.cache.processor.EntryProcessorResult; import javax.cache.processor.MutableEntry; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; import junit.framework.AssertionFailedError; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; -import org.apache.ignite.IgniteLogger; import org.apache.ignite.IgniteTransactions; import org.apache.ignite.cache.CacheEntry; import org.apache.ignite.cache.CacheEntryEventSerializableFilter; @@ -83,7 +82,6 @@ import org.apache.ignite.lang.IgniteClosure; import org.apache.ignite.lang.IgniteFuture; import org.apache.ignite.lang.IgnitePredicate; -import org.apache.ignite.resources.LoggerResource; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.IgniteCacheConfigVariationsAbstractTest; import org.apache.ignite.transactions.Transaction; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerAbstractTest.java index bcf46fdd1ef39..e473d5238289b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerAbstractTest.java @@ -65,7 +65,6 @@ import org.apache.ignite.internal.util.lang.GridAbsPredicate; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; import org.apache.ignite.spi.eventstorage.memory.MemoryEventStorageSpi; import org.apache.ignite.testframework.GridTestUtils; import org.jetbrains.annotations.Nullable; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java index d3269c30da6af..b55e3d0b21346 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java @@ -17,7 +17,6 @@ package org.apache.ignite.internal.processors.cache; -import com.google.common.collect.Sets; import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; @@ -50,6 +49,7 @@ import javax.cache.integration.CacheWriterException; import javax.cache.processor.EntryProcessorException; import javax.cache.processor.MutableEntry; +import com.google.common.collect.Sets; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; @@ -78,8 +78,8 @@ import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.IgniteKernal; import org.apache.ignite.internal.binary.BinaryMarshaller; -import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; import org.apache.ignite.internal.processors.platform.cache.expiry.PlatformExpiryPolicyFactory; import org.apache.ignite.internal.util.lang.GridAbsPredicate; import org.apache.ignite.internal.util.lang.GridIterator; @@ -87,6 +87,7 @@ import org.apache.ignite.internal.util.lang.gridfunc.ContainsPredicate; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.PA; +import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiInClosure; @@ -2892,10 +2893,10 @@ public void testConcurrentOperationsAndCacheDestroy() throws Exception { IgniteInternalFuture cacheFut = GridTestUtils.runAsync(new Runnable() { @Override public void run() { - try { - int cntr = 0; + int cntr = 0; - while (!stop.get()) { + while (!stop.get()) { + try { ThreadLocalRandom rnd = ThreadLocalRandom.current(); String grp; @@ -2927,13 +2928,20 @@ public void testConcurrentOperationsAndCacheDestroy() throws Exception { node.destroyCache(cache.getName()); } - } - catch (Exception e) { - err.set(true); + catch (Exception e) { + if (X.hasCause(e, CacheStoppedException.class)) { + // Cache operation can be blocked on + // awaiting new topology version and cancelled with CacheStoppedException cause. - log.error("Unexpected error(2): " + e, e); + continue; + } - stop.set(true); + err.set(true); + + log.error("Unexpected error(2): " + e, e); + + stop.set(true); + } } } }, "cache-destroy-thread"); @@ -3706,7 +3714,7 @@ public void testRestartsAndCacheCreateDestroy() throws Exception { final AtomicReferenceArray caches = new AtomicReferenceArray<>(CACHES); - for (int i = 0; i < 10; i++) { + for (int i = 0; i < CACHES; i++) { CacheAtomicityMode atomicityMode = i % 2 == 0 ? ATOMIC : TRANSACTIONAL; caches.set(i, @@ -3799,28 +3807,41 @@ public void testRestartsAndCacheCreateDestroy() throws Exception { IgniteInternalFuture opFut = GridTestUtils.runMultiThreadedAsync(new Runnable() { @Override public void run() { - try { - ThreadLocalRandom rnd = ThreadLocalRandom.current(); + ThreadLocalRandom rnd = ThreadLocalRandom.current(); - while (!stop.get()) { + while (!stop.get()) { + try { int idx = rnd.nextInt(CACHES); IgniteCache cache = caches.get(idx); if (cache != null && caches.compareAndSet(idx, cache, null)) { - for (int i = 0; i < 10; i++) - cacheOperation(rnd, cache); - - caches.set(idx, cache); + try { + for (int i = 0; i < 10; i++) + cacheOperation(rnd, cache); + } + catch (Exception e) { + if (X.hasCause(e, CacheStoppedException.class)) { + // Cache operation can be blocked on + // awaiting new topology version and cancelled with CacheStoppedException cause. + + continue; + } + + throw e; + } + finally { + caches.set(idx, cache); + } } } - } - catch (Exception e) { - err.set(e); + catch (Exception e) { + err.set(e); - log.error("Unexpected error: " + e, e); + log.error("Unexpected error: " + e, e); - stop.set(true); + stop.set(true); + } } } }, 8, "op-thread"); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheStartStopLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheStartStopLoadTest.java index 7cb9861c5e11f..25b90c4688829 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheStartStopLoadTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheStartStopLoadTest.java @@ -113,7 +113,6 @@ public void testMemoryLeaks() throws Exception { cache.put(1, obj); - weakMap.put(((IgniteCacheProxy)cache).delegate(), Boolean.TRUE); weakMap.put(obj, Boolean.TRUE); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/MemoryPolicyInitializationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/MemoryPolicyInitializationTest.java index 9a49b6c6cd46f..6493f88b29198 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/MemoryPolicyInitializationTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/MemoryPolicyInitializationTest.java @@ -24,6 +24,7 @@ import org.apache.ignite.configuration.MemoryPolicyConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.processors.cache.GridCacheContext; +import org.apache.ignite.internal.processors.cache.IgniteCacheProxy; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; @@ -217,7 +218,7 @@ public void testCachesOnUserDefinedDefaultMemoryPolicy() throws Exception { * @param plcName Policy name. */ private void verifyCacheMemoryPolicy(IgniteCache cache, String plcName) { - GridCacheContext ctx = U.field(cache, "ctx"); + GridCacheContext ctx = ((IgniteCacheProxy) cache).context(); assertEquals(plcName, ctx.memoryPolicy().config().getName()); } diff --git a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerAbstractTest.java index 66f0d72cc57f6..05a8924d6186e 100644 --- a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerAbstractTest.java @@ -56,7 +56,7 @@ import org.apache.ignite.internal.cluster.ClusterGroupAdapter; import org.apache.ignite.internal.cluster.ClusterNodeLocalMapImpl; import org.apache.ignite.internal.executor.GridExecutorService; -import org.apache.ignite.internal.processors.cache.IgniteCacheProxy; +import org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy; import org.apache.ignite.internal.processors.service.DummyService; import org.apache.ignite.internal.util.GridByteArrayList; import org.apache.ignite.internal.util.typedef.F; @@ -171,8 +171,8 @@ public void testDefaultCache() throws Exception { assert inBean.getObjectField() != null; assert outBean.getObjectField() != null; - assert inBean.getObjectField().getClass().equals(IgniteCacheProxy.class); - assert outBean.getObjectField().getClass().equals(IgniteCacheProxy.class); + assert inBean.getObjectField().getClass().equals(GatewayProtectedCacheProxy.class); + assert outBean.getObjectField().getClass().equals(GatewayProtectedCacheProxy.class); assert inBean != outBean; @@ -201,8 +201,8 @@ public void testNamedCache() throws Exception { assert inBean.getObjectField() != null; assert outBean.getObjectField() != null; - assert inBean.getObjectField().getClass().equals(IgniteCacheProxy.class); - assert outBean.getObjectField().getClass().equals(IgniteCacheProxy.class); + assert inBean.getObjectField().getClass().equals(GatewayProtectedCacheProxy.class); + assert outBean.getObjectField().getClass().equals(GatewayProtectedCacheProxy.class); assert inBean != outBean; From 9825eaa35c5c95485f7d7c3b60de7d0ca47dad23 Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Thu, 27 Jul 2017 10:42:30 +0300 Subject: [PATCH 039/547] IGNITE-5761 - Fixed header license --- .../dht/NotMappedPartitionInTxTest.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/NotMappedPartitionInTxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/NotMappedPartitionInTxTest.java index 4059660ca5948..e09cf53c51c11 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/NotMappedPartitionInTxTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/NotMappedPartitionInTxTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ignite.internal.processors.cache.distributed.dht; import java.util.Collection; From 004e5aa7df72a16efb02e7500b56475916515ba7 Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Thu, 27 Jul 2017 11:42:09 +0300 Subject: [PATCH 040/547] IGNITE-5729 - Removed RendezvousAffinityFunction with extra partitions in GridCacheAbstractSelfTest --- .../internal/processors/cache/GridCacheAbstractSelfTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java index 9376971a8a7ae..81aadb93c7d7e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java @@ -247,7 +247,6 @@ protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throw } } - cfg.setAffinity(new RendezvousAffinityFunction(false, 4096)); cfg.setCacheMode(cacheMode()); cfg.setAtomicityMode(atomicityMode()); cfg.setWriteSynchronizationMode(writeSynchronization()); From 5704e393102cc8c24df7bfb4ff9053003530b7fc Mon Sep 17 00:00:00 2001 From: sboikov Date: Thu, 27 Jul 2017 12:50:37 +0300 Subject: [PATCH 041/547] Tried to simplify GridDhtAtomicCache.updateAllAsyncInternal0. --- .../dht/atomic/DhtAtomicUpdateResult.java | 131 ++++++ .../dht/atomic/GridDhtAtomicCache.java | 380 +++++++----------- .../GridNearAtomicAbstractUpdateFuture.java | 2 +- .../atomic/GridNearAtomicUpdateFuture.java | 4 +- 4 files changed, 282 insertions(+), 235 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/DhtAtomicUpdateResult.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/DhtAtomicUpdateResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/DhtAtomicUpdateResult.java new file mode 100644 index 0000000000000..e7d2b1996a105 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/DhtAtomicUpdateResult.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed.dht.atomic; + +import java.util.ArrayList; +import java.util.Collection; +import org.apache.ignite.internal.processors.cache.GridCacheReturn; +import org.apache.ignite.internal.processors.cache.GridCacheUpdateAtomicResult; +import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry; +import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.lang.IgniteBiTuple; +import org.jetbrains.annotations.Nullable; + +/** + * + */ +class DhtAtomicUpdateResult { + /** */ + private GridCacheReturn retVal; + + /** */ + private Collection> deleted; + + /** */ + private GridDhtAtomicAbstractUpdateFuture dhtFut; + + /** */ + private IgniteCacheExpiryPolicy expiry; + + /** + * + */ + DhtAtomicUpdateResult() { + // No-op. + } + + /** + * @param retVal Return value. + * @param deleted Deleted entries. + * @param dhtFut DHT update future. + */ + DhtAtomicUpdateResult(GridCacheReturn retVal, + Collection> deleted, + GridDhtAtomicAbstractUpdateFuture dhtFut) { + this.retVal = retVal; + this.deleted = deleted; + this.dhtFut = dhtFut; + } + + /** + * @param expiry Expiry policy. + */ + void expiryPolicy(@Nullable IgniteCacheExpiryPolicy expiry) { + this.expiry = expiry; + } + + /** + * @return Expiry policy. + */ + @Nullable IgniteCacheExpiryPolicy expiryPolicy() { + return expiry; + } + + /** + * @param entry Entry. + * @param updRes Entry update result. + * @param entries All entries. + */ + void addDeleted(GridDhtCacheEntry entry, + GridCacheUpdateAtomicResult updRes, + Collection entries) { + if (updRes.removeVersion() != null) { + if (deleted == null) + deleted = new ArrayList<>(entries.size()); + + deleted.add(F.t(entry, updRes.removeVersion())); + } + } + + /** + * @return Deleted entries. + */ + Collection> deleted() { + return deleted; + } + + /** + * @return DHT future. + */ + GridDhtAtomicAbstractUpdateFuture dhtFuture() { + return dhtFut; + } + + /** + * @param retVal Result for operation. + */ + void returnValue(GridCacheReturn retVal) { + this.retVal = retVal; + } + + /** + * @return Result for invoke operation. + */ + GridCacheReturn returnValue() { + return retVal; + } + + /** + * @param dhtFut DHT future. + */ + void dhtFuture(@Nullable GridDhtAtomicAbstractUpdateFuture dhtFut) { + this.dhtFut = dhtFut; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java index 712babd999333..be4aacee08993 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java @@ -1658,12 +1658,12 @@ else if (!skipVals && ctx.config().isStatisticsEnabled()) /** * Executes local update. * - * @param nodeId Node ID. + * @param node Node. * @param req Update request. * @param completionCb Completion callback. */ void updateAllAsyncInternal( - final UUID nodeId, + final ClusterNode node, final GridNearAtomicAbstractUpdateRequest req, final UpdateReplyClosure completionCb ) { @@ -1678,12 +1678,12 @@ void updateAllAsyncInternal( return; } catch (IgniteCheckedException e) { - onForceKeysError(nodeId, req, completionCb, e); + onForceKeysError(node.id(), req, completionCb, e); return; } - updateAllAsyncInternal0(nodeId, req, completionCb); + updateAllAsyncInternal0(node, req, completionCb); } else { forceFut.listen(new CI1>() { @@ -1695,12 +1695,12 @@ void updateAllAsyncInternal( return; } catch (IgniteCheckedException e) { - onForceKeysError(nodeId, req, completionCb, e); + onForceKeysError(node.id(), req, completionCb, e); return; } - updateAllAsyncInternal0(nodeId, req, completionCb); + updateAllAsyncInternal0(node, req, completionCb); } }); } @@ -1732,26 +1732,17 @@ private void onForceKeysError(final UUID nodeId, /** * Executes local update after preloader fetched values. * - * @param nodeId Node ID. + * @param node Node. * @param req Update request. * @param completionCb Completion callback. */ private void updateAllAsyncInternal0( - UUID nodeId, + ClusterNode node, GridNearAtomicAbstractUpdateRequest req, UpdateReplyClosure completionCb ) { - ClusterNode node = ctx.discovery().node(nodeId); - - if (node == null) { - U.warn(msgLog, "Skip near update request, node originated update request left [" + - "futId=" + req.futureId() + ", node=" + nodeId + ']'); - - return; - } - GridNearAtomicUpdateResponse res = new GridNearAtomicUpdateResponse(ctx.cacheId(), - nodeId, + node.id(), req.futureId(), req.partition(), false, @@ -1763,8 +1754,6 @@ private void updateAllAsyncInternal0( boolean remap = false; - String taskName = ctx.kernalContext().task().resolveTaskName(req.taskNameHash()); - IgniteCacheExpiryPolicy expiry = null; ctx.shared().database().checkpointReadLock(); @@ -1795,97 +1784,11 @@ private void updateAllAsyncInternal0( // Do not check topology version if topology was locked on near node by // external transaction or explicit lock. if (req.topologyLocked() || !needRemap(req.topologyVersion(), top.topologyVersion())) { - boolean hasNear = req.nearCache(); - - // Assign next version for update inside entries lock. - GridCacheVersion ver = ctx.versions().next(top.topologyVersion()); - - if (hasNear) - res.nearVersion(ver); - - if (msgLog.isDebugEnabled()) { - msgLog.debug("Assigned update version [futId=" + req.futureId() + - ", writeVer=" + ver + ']'); - } - - assert ver != null : "Got null version for update request: " + req; - - boolean sndPrevVal = !top.rebalanceFinished(req.topologyVersion()); - - dhtFut = createDhtFuture(ver, req); - - expiry = expiryPolicy(req.expiry()); - - GridCacheReturn retVal = null; - - if (req.size() > 1 && // Several keys ... - writeThrough() && !req.skipStore() && // and store is enabled ... - !ctx.store().isLocal() && // and this is not local store ... - // (conflict resolver should be used for local store) - !ctx.dr().receiveEnabled() // and no DR. - ) { - // This method can only be used when there are no replicated entries in the batch. - UpdateBatchResult updRes = updateWithBatch(node, - hasNear, - req, - res, - locked, - ver, - dhtFut, - ctx.isDrEnabled(), - taskName, - expiry, - sndPrevVal); - - deleted = updRes.deleted(); - dhtFut = updRes.dhtFuture(); - - if (req.operation() == TRANSFORM) - retVal = updRes.invokeResults(); - } - else { - UpdateSingleResult updRes = updateSingle(node, - hasNear, - req, - res, - locked, - ver, - dhtFut, - ctx.isDrEnabled(), - taskName, - expiry, - sndPrevVal); - - retVal = updRes.returnValue(); - deleted = updRes.deleted(); - dhtFut = updRes.dhtFuture(); - } + DhtAtomicUpdateResult updRes = update(node, locked, req, res); - if (retVal == null) - retVal = new GridCacheReturn(ctx, node.isLocal(), true, null, true); - - res.returnValue(retVal); - - if (dhtFut != null) { - if (req.writeSynchronizationMode() == PRIMARY_SYNC - // To avoid deadlock disable back-pressure for sender data node. - && !ctx.discovery().cacheAffinityNode(node, ctx.name()) - && !dhtFut.isDone()) { - final IgniteRunnable tracker = GridNioBackPressureControl.threadTracker(); - - if (tracker != null && tracker instanceof GridNioMessageTracker) { - ((GridNioMessageTracker)tracker).onMessageReceived(); - - dhtFut.listen(new IgniteInClosure>() { - @Override public void apply(IgniteInternalFuture fut) { - ((GridNioMessageTracker)tracker).onMessageProcessed(); - } - }); - } - } - - ctx.mvcc().addAtomicFuture(dhtFut.id(), dhtFut); - } + dhtFut = updRes.dhtFuture(); + deleted = updRes.deleted(); + expiry = updRes.expiryPolicy(); } else { // Should remap all keys. @@ -1953,9 +1856,10 @@ private void updateAllAsyncInternal0( completionCb.apply(req, res); } - else + else { if (dhtFut != null) dhtFut.map(node, res.returnValue(), res, completionCb); + } if (req.writeSynchronizationMode() != FULL_ASYNC) req.cleanup(!node.isLocal()); @@ -1963,6 +1867,122 @@ private void updateAllAsyncInternal0( sendTtlUpdateRequest(expiry); } + /** + * @param node Node. + * @param locked Entries. + * @param req Request. + * @param res Response. + * @return Operation result. + * @throws GridCacheEntryRemovedException If got obsolete entry. + */ + private DhtAtomicUpdateResult update( + ClusterNode node, + List locked, + GridNearAtomicAbstractUpdateRequest req, + GridNearAtomicUpdateResponse res) + throws GridCacheEntryRemovedException + { + GridDhtPartitionTopology top = topology(); + + String taskName = ctx.kernalContext().task().resolveTaskName(req.taskNameHash()); + + boolean hasNear = req.nearCache(); + + // Assign next version for update inside entries lock. + GridCacheVersion ver = ctx.versions().next(top.topologyVersion()); + + if (hasNear) + res.nearVersion(ver); + + if (msgLog.isDebugEnabled()) { + msgLog.debug("Assigned update version [futId=" + req.futureId() + + ", writeVer=" + ver + ']'); + } + + assert ver != null : "Got null version for update request: " + req; + + boolean sndPrevVal = !top.rebalanceFinished(req.topologyVersion()); + + GridDhtAtomicAbstractUpdateFuture dhtFut = createDhtFuture(ver, req); + + IgniteCacheExpiryPolicy expiry = expiryPolicy(req.expiry()); + + GridCacheReturn retVal = null; + + DhtAtomicUpdateResult updRes; + + if (req.size() > 1 && // Several keys ... + writeThrough() && !req.skipStore() && // and store is enabled ... + !ctx.store().isLocal() && // and this is not local store ... + // (conflict resolver should be used for local store) + !ctx.dr().receiveEnabled() // and no DR. + ) { + // This method can only be used when there are no replicated entries in the batch. + updRes = updateWithBatch(node, + hasNear, + req, + res, + locked, + ver, + dhtFut, + ctx.isDrEnabled(), + taskName, + expiry, + sndPrevVal); + + dhtFut = updRes.dhtFuture(); + + if (req.operation() == TRANSFORM) + retVal = updRes.returnValue(); + } + else { + updRes = updateSingle(node, + hasNear, + req, + res, + locked, + ver, + dhtFut, + ctx.isDrEnabled(), + taskName, + expiry, + sndPrevVal); + + retVal = updRes.returnValue(); + dhtFut = updRes.dhtFuture(); + } + + if (retVal == null) + retVal = new GridCacheReturn(ctx, node.isLocal(), true, null, true); + + res.returnValue(retVal); + + if (dhtFut != null) { + if (req.writeSynchronizationMode() == PRIMARY_SYNC + // To avoid deadlock disable back-pressure for sender data node. + && !ctx.discovery().cacheAffinityNode(node, ctx.name()) + && !dhtFut.isDone()) { + final IgniteRunnable tracker = GridNioBackPressureControl.threadTracker(); + + if (tracker != null && tracker instanceof GridNioMessageTracker) { + ((GridNioMessageTracker)tracker).onMessageReceived(); + + dhtFut.listen(new IgniteInClosure>() { + @Override public void apply(IgniteInternalFuture fut) { + ((GridNioMessageTracker)tracker).onMessageProcessed(); + } + }); + } + } + + ctx.mvcc().addAtomicFuture(dhtFut.id(), dhtFut); + } + + updRes.expiryPolicy(expiry); + + return updRes; + } + /** * Updates locked entries using batched write-through. * @@ -1981,7 +2001,7 @@ private void updateAllAsyncInternal0( * @throws GridCacheEntryRemovedException Should not be thrown. */ @SuppressWarnings("unchecked") - private UpdateBatchResult updateWithBatch( + private DhtAtomicUpdateResult updateWithBatch( final ClusterNode node, final boolean hasNear, final GridNearAtomicAbstractUpdateRequest req, @@ -2004,7 +2024,7 @@ private UpdateBatchResult updateWithBatch( catch (IgniteCheckedException e) { res.addFailedKeys(req.keys(), e); - return new UpdateBatchResult(); + return new DhtAtomicUpdateResult(); } } @@ -2018,7 +2038,7 @@ private UpdateBatchResult updateWithBatch( List writeVals = null; - UpdateBatchResult updRes = new UpdateBatchResult(); + DhtAtomicUpdateResult updRes = new DhtAtomicUpdateResult(); List filtered = new ArrayList<>(size); @@ -2317,7 +2337,7 @@ else if (op == UPDATE) { updRes.dhtFuture(dhtFut); - updRes.invokeResult(invokeRes); + updRes.returnValue(invokeRes); return updRes; } @@ -2390,7 +2410,7 @@ private void reloadIfNeeded(final List entries) throws Ignite * @return Return value. * @throws GridCacheEntryRemovedException Should be never thrown. */ - private UpdateSingleResult updateSingle( + private DhtAtomicUpdateResult updateSingle( ClusterNode nearNode, boolean hasNear, GridNearAtomicAbstractUpdateRequest req, @@ -2577,7 +2597,7 @@ else if (GridDhtCacheEntry.ReaderId.contains(readers, nearNode.id())) { } } - return new UpdateSingleResult(retVal, deleted, dhtFut); + return new DhtAtomicUpdateResult(retVal, deleted, dhtFut); } /** @@ -2615,7 +2635,7 @@ else if (GridDhtCacheEntry.ReaderId.contains(readers, nearNode.id())) { final GridNearAtomicAbstractUpdateRequest req, final GridNearAtomicUpdateResponse res, final boolean replicate, - final UpdateBatchResult batchRes, + final DhtAtomicUpdateResult batchRes, final String taskName, @Nullable final IgniteCacheExpiryPolicy expiry, final boolean sndPrevVal @@ -3060,7 +3080,16 @@ private void processNearAtomicUpdateRequest(UUID nodeId, GridNearAtomicAbstractU ", node=" + nodeId + ']'); } - updateAllAsyncInternal(nodeId, req, updateReplyClos); + ClusterNode node = ctx.discovery().node(nodeId); + + if (node == null) { + U.warn(msgLog, "Skip near update request, node originated update request left [" + + "futId=" + req.futureId() + ", node=" + nodeId + ']'); + + return; + } + + updateAllAsyncInternal(node, req, updateReplyClos); } /** @@ -3540,119 +3569,6 @@ private void sendNearUpdateReply(UUID nodeId, GridNearAtomicUpdateResponse res) return S.toString(GridDhtAtomicCache.class, this, super.toString()); } - /** - * Result of {@link GridDhtAtomicCache#updateSingle} execution. - */ - private static class UpdateSingleResult { - /** */ - private final GridCacheReturn retVal; - - /** */ - private final Collection> deleted; - - /** */ - private final GridDhtAtomicAbstractUpdateFuture dhtFut; - - /** - * @param retVal Return value. - * @param deleted Deleted entries. - * @param dhtFut DHT future. - */ - private UpdateSingleResult(GridCacheReturn retVal, - Collection> deleted, - GridDhtAtomicAbstractUpdateFuture dhtFut) { - this.retVal = retVal; - this.deleted = deleted; - this.dhtFut = dhtFut; - } - - /** - * @return Return value. - */ - private GridCacheReturn returnValue() { - return retVal; - } - - /** - * @return Deleted entries. - */ - private Collection> deleted() { - return deleted; - } - - /** - * @return DHT future. - */ - public GridDhtAtomicAbstractUpdateFuture dhtFuture() { - return dhtFut; - } - } - - /** - * Result of {@link GridDhtAtomicCache#updateWithBatch} execution. - */ - private static class UpdateBatchResult { - /** */ - private Collection> deleted; - - /** */ - private GridDhtAtomicAbstractUpdateFuture dhtFut; - - /** */ - private GridCacheReturn invokeRes; - - /** - * @param entry Entry. - * @param updRes Entry update result. - * @param entries All entries. - */ - private void addDeleted(GridDhtCacheEntry entry, - GridCacheUpdateAtomicResult updRes, - Collection entries) { - if (updRes.removeVersion() != null) { - if (deleted == null) - deleted = new ArrayList<>(entries.size()); - - deleted.add(F.t(entry, updRes.removeVersion())); - } - } - - /** - * @return Deleted entries. - */ - private Collection> deleted() { - return deleted; - } - - /** - * @return DHT future. - */ - public GridDhtAtomicAbstractUpdateFuture dhtFuture() { - return dhtFut; - } - - /** - * @param invokeRes Result for invoke operation. - */ - private void invokeResult(GridCacheReturn invokeRes) { - this.invokeRes = invokeRes; - } - - /** - * @return Result for invoke operation. - */ - GridCacheReturn invokeResults() { - return invokeRes; - } - - /** - * @param dhtFut DHT future. - */ - private void dhtFuture(@Nullable GridDhtAtomicAbstractUpdateFuture dhtFut) { - this.dhtFut = dhtFut; - } - } - /** * */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java index 6fe96a437a56e..983b18ac38ba0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java @@ -296,7 +296,7 @@ final boolean storeFuture() { */ final void sendSingleRequest(UUID nodeId, GridNearAtomicAbstractUpdateRequest req) { if (cctx.localNodeId().equals(nodeId)) { - cache.updateAllAsyncInternal(nodeId, req, + cache.updateAllAsyncInternal(cctx.localNode(), req, new GridDhtAtomicCache.UpdateReplyClosure() { @Override public void apply(GridNearAtomicAbstractUpdateRequest req, GridNearAtomicUpdateResponse res) { if (syncMode != FULL_ASYNC) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java index 138645d86bcf6..930012ab1cb85 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java @@ -706,7 +706,7 @@ private void sendUpdateRequests(Map mappings) { } if (locUpdate != null) { - cache.updateAllAsyncInternal(cctx.localNodeId(), locUpdate, + cache.updateAllAsyncInternal(cctx.localNode(), locUpdate, new GridDhtAtomicCache.UpdateReplyClosure() { @Override public void apply(GridNearAtomicAbstractUpdateRequest req, GridNearAtomicUpdateResponse res) { if (syncMode != FULL_ASYNC) @@ -730,7 +730,7 @@ else if (res.remapTopologyVersion() != null) * @param topVer Topology version. * @param remapKeys Keys to remap. */ - void map(AffinityTopologyVersion topVer, @Nullable Collection remapKeys) { + private void map(AffinityTopologyVersion topVer, @Nullable Collection remapKeys) { Collection topNodes = CU.affinityNodes(cctx, topVer); if (F.isEmpty(topNodes)) { From 2574bebde0fc5591bc09e2241f1352e1f0e9cbca Mon Sep 17 00:00:00 2001 From: sboikov Date: Thu, 27 Jul 2017 13:21:14 +0300 Subject: [PATCH 042/547] Removed unused methods from IgniteThreadPoolExecutor. --- .../apache/ignite/internal/IgnitionEx.java | 3 +- .../internal/processors/igfs/IgfsImpl.java | 2 +- .../thread/IgniteThreadPoolExecutor.java | 144 +----------------- .../loadtests/colocation/GridTestMain.java | 45 ------ .../GridMarshallerResourceBean.java | 5 +- ...GridThreadPoolExecutorServiceSelfTest.java | 7 +- 6 files changed, 10 insertions(+), 196 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java index d219333ea958c..1139ec60860d0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java @@ -1776,8 +1776,7 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException { cfg.getIgfsThreadPoolSize(), DFLT_THREAD_KEEP_ALIVE_TIME, new LinkedBlockingQueue(), - new IgfsThreadFactory(cfg.getIgniteInstanceName(), "igfs"), - null /* Abort policy will be used. */); + new IgfsThreadFactory(cfg.getIgniteInstanceName(), "igfs")); igfsExecSvc.allowCoreThreadTimeOut(true); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java index 7eb61d10366b3..5808e7cc4a620 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java @@ -247,7 +247,7 @@ public final class IgfsImpl implements IgfsEx { } dualPool = secondaryFs != null ? new IgniteThreadPoolExecutor(4, Integer.MAX_VALUE, 5000L, - new SynchronousQueue(), new IgfsThreadFactory(cfg.getName()), null) : null; + new SynchronousQueue(), new IgfsThreadFactory(cfg.getName())) : null; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadPoolExecutor.java b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadPoolExecutor.java index 8002aaa5d1069..639ef9438a669 100644 --- a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadPoolExecutor.java +++ b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadPoolExecutor.java @@ -19,150 +19,14 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingDeque; -import java.util.concurrent.RejectedExecutionHandler; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import org.jetbrains.annotations.Nullable; /** * An {@link ExecutorService} that executes submitted tasks using pooled grid threads. */ public class IgniteThreadPoolExecutor extends ThreadPoolExecutor { - /** Default core pool size (value is {@code 100}). */ - public static final int DFLT_CORE_POOL_SIZE = 100; - - /** - * Creates a new service with default initial parameters. - * Default values are: - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
NameDefault Value
Core Pool Size{@code 100} (see {@link #DFLT_CORE_POOL_SIZE}).
Maximum Pool SizeNone, is it is not used for unbounded queues.
Keep alive timeNo limit (see {@link Long#MAX_VALUE}).
Blocking Queue (see {@link BlockingQueue}).Unbounded linked blocking queue (see {@link LinkedBlockingDeque}).
- */ - public IgniteThreadPoolExecutor() { - this( - DFLT_CORE_POOL_SIZE, - DFLT_CORE_POOL_SIZE, - 0, - new LinkedBlockingDeque(), - new IgniteThreadFactory(null), - null - ); - } - - /** - * Creates a new service with the given initial parameters. - * - * @param corePoolSize The number of threads to keep in the pool, even if they are idle. - * @param maxPoolSize The maximum number of threads to allow in the pool. - * @param keepAliveTime When the number of threads is greater than the core, this is the maximum time - * that excess idle threads will wait for new tasks before terminating. - * @param workQueue The queue to use for holding tasks before they are executed. This queue will hold only - * runnable tasks submitted by the {@link #execute(Runnable)} method. - */ - public IgniteThreadPoolExecutor( - int corePoolSize, - int maxPoolSize, - long keepAliveTime, - BlockingQueue workQueue) { - this( - corePoolSize, - maxPoolSize, - keepAliveTime, - workQueue, - new IgniteThreadFactory(null), - null - ); - } - - /** - * Creates a new service with the given initial parameters. - * - * @param corePoolSize The number of threads to keep in the pool, even if they are idle. - * @param maxPoolSize The maximum number of threads to allow in the pool. - * @param keepAliveTime When the number of threads is greater than the core, this is the maximum time - * that excess idle threads will wait for new tasks before terminating. - * @param workQ The queue to use for holding tasks before they are executed. This queue will hold only the - * runnable tasks submitted by the {@link #execute(Runnable)} method. - * @param hnd Optional handler to use when execution is blocked because the thread bounds and queue - * capacities are reached. If {@code null} then {@code AbortPolicy} - * handler is used by default. - */ - public IgniteThreadPoolExecutor( - int corePoolSize, - int maxPoolSize, - long keepAliveTime, - BlockingQueue workQ, - RejectedExecutionHandler hnd) { - this( - corePoolSize, - maxPoolSize, - keepAliveTime, - workQ, - new IgniteThreadFactory(null), - hnd - ); - } - - /** - * Creates a new service with default initial parameters. - * Default values are: - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
NameDefault Value
Core Pool Size{@code 100} (see {@link #DFLT_CORE_POOL_SIZE}).
Maximum Pool SizeNone, is it is not used for unbounded queues.
Keep alive timeNo limit (see {@link Long#MAX_VALUE}).
Blocking Queue (see {@link BlockingQueue}).Unbounded linked blocking queue (see {@link LinkedBlockingDeque}).
- * - * @param igniteInstanceName Name of the grid. - */ - public IgniteThreadPoolExecutor(String igniteInstanceName) { - this( - DFLT_CORE_POOL_SIZE, - DFLT_CORE_POOL_SIZE, - 0, - new LinkedBlockingDeque(), - new IgniteThreadFactory(igniteInstanceName), - null - ); - } - /** * Creates a new service with the given initial parameters. * @@ -202,17 +66,13 @@ public IgniteThreadPoolExecutor( * @param workQ The queue to use for holding tasks before they are executed. This queue will hold only the * runnable tasks submitted by the {@link #execute(Runnable)} method. * @param threadFactory Thread factory. - * @param hnd Optional handler to use when execution is blocked because the thread bounds and queue - * capacities are reached. If {@code null} then {@code AbortPolicy} - * handler is used by default. */ public IgniteThreadPoolExecutor( int corePoolSize, int maxPoolSize, long keepAliveTime, BlockingQueue workQ, - ThreadFactory threadFactory, - @Nullable RejectedExecutionHandler hnd) { + ThreadFactory threadFactory) { super( corePoolSize, maxPoolSize, @@ -220,7 +80,7 @@ public IgniteThreadPoolExecutor( TimeUnit.MILLISECONDS, workQ, threadFactory, - hnd == null ? new AbortPolicy() : hnd + new AbortPolicy() ); } } \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/loadtests/colocation/GridTestMain.java b/modules/core/src/test/java/org/apache/ignite/loadtests/colocation/GridTestMain.java index bf345452e3ecb..aa0764e2a35fd 100644 --- a/modules/core/src/test/java/org/apache/ignite/loadtests/colocation/GridTestMain.java +++ b/modules/core/src/test/java/org/apache/ignite/loadtests/colocation/GridTestMain.java @@ -118,51 +118,6 @@ private static void colocateJobs() throws Exception { X.println("Executed " + GridTestConstants.ENTRY_COUNT + " computations in " + (end - start) + "ms."); } - /** - * - */ - private static void localPoolRun() { - X.println("Local thread pool run..."); - - ExecutorService exe = new IgniteThreadPoolExecutor(400, 400, 0, new ArrayBlockingQueue(400) { - @Override public boolean offer(Runnable runnable) { - try { - put(runnable); - } - catch (InterruptedException e) { - e.printStackTrace(); - } - - return true; - } - }); - - long start = System.currentTimeMillis(); - - final IgniteCache cache = G.ignite().cache("partitioned"); - - // Collocate computations and data. - for (long i = 0; i < GridTestConstants.ENTRY_COUNT; i++) { - final long key = i; - - exe.submit(new Runnable() { - @Override public void run() { - Long val = cache.localPeek(new GridTestKey(key), CachePeekMode.ONHEAP); - - if (val == null || val != key) - throw new RuntimeException("Invalid value found [key=" + key + ", val=" + val + ']'); - } - }); - - if (i % 10000 == 0) - X.println("Executed jobs: " + i); - } - - long end = System.currentTimeMillis(); - - X.println("Executed " + GridTestConstants.ENTRY_COUNT + " computations in " + (end - start) + "ms."); - } - /** * Load cache from data store. Also take a look at * {@link GridTestCacheStore#loadAll} method. diff --git a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerResourceBean.java b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerResourceBean.java index a0bdf7e92ae73..0887879b52f04 100644 --- a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerResourceBean.java +++ b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerResourceBean.java @@ -21,7 +21,7 @@ import java.lang.management.ManagementFactory; import java.util.Collection; import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.Executors; import javax.management.MBeanServer; import org.apache.ignite.GridTestJobContext; import org.apache.ignite.GridTestTaskSession; @@ -74,7 +74,7 @@ class GridMarshallerResourceBean implements Serializable { marshaller = new JdkMarshaller(); mbeanSrv = ManagementFactory.getPlatformMBeanServer(); ses = new GridTestTaskSession(); - execSvc = new IgniteThreadPoolExecutor(1, 1, 0, new LinkedBlockingQueue()); + execSvc = Executors.newSingleThreadExecutor(); appCtx = new GenericApplicationContext(); jobCtx = new GridTestJobContext(); balancer = new LoadBalancer(); @@ -98,6 +98,7 @@ public void checkNullResources() { private static class LoadBalancer extends GridLoadBalancerAdapter { /** */ public LoadBalancer() { + // No-op. } /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/thread/GridThreadPoolExecutorServiceSelfTest.java b/modules/core/src/test/java/org/apache/ignite/thread/GridThreadPoolExecutorServiceSelfTest.java index bad42b0a9be41..3948f6a620bc6 100644 --- a/modules/core/src/test/java/org/apache/ignite/thread/GridThreadPoolExecutorServiceSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/thread/GridThreadPoolExecutorServiceSelfTest.java @@ -85,7 +85,7 @@ public void testSingleGridThreadExecutor() throws Exception { * @throws ExecutionException If failed. */ public void testGridThreadPoolExecutor() throws Exception { - IgniteThreadPoolExecutor exec = new IgniteThreadPoolExecutor(1, 1, 0, new LinkedBlockingQueue()); + IgniteThreadPoolExecutor exec = new IgniteThreadPoolExecutor("", "", 1, 1, 0, new LinkedBlockingQueue()); exec.submit(new InterruptingRunnable()).get(); @@ -101,7 +101,7 @@ public void testGridThreadPoolExecutor() throws Exception { * @throws ExecutionException If failed. */ public void testGridThreadPoolExecutorRejection() throws Exception { - IgniteThreadPoolExecutor exec = new IgniteThreadPoolExecutor(1, 1, 0, new LinkedBlockingQueue()); + IgniteThreadPoolExecutor exec = new IgniteThreadPoolExecutor("", "", 1, 1, 0, new LinkedBlockingQueue()); for (int i = 0; i < 10; i++) exec.submit(new TestRunnable()); @@ -141,8 +141,7 @@ public void testGridThreadPoolExecutorPrestartCoreThreads() throws Exception { } }); } - }, - null + } ); assert exec.prestartAllCoreThreads() == THREAD_CNT; From be5a9eaf77877b02a46f30730cd6d8e1e3919f41 Mon Sep 17 00:00:00 2001 From: Sergey Chugunov Date: Thu, 27 Jul 2017 13:02:59 +0300 Subject: [PATCH 043/547] GG-12485 added test to verify issue on 8.1.x version --- ...bSingleNodeWithIndexingWalRestoreTest.java | 197 ++++++++++++++++++ .../IgnitePdsWithIndexingTestSuite.java | 2 + 2 files changed, 199 insertions(+) create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodeWithIndexingWalRestoreTest.java diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodeWithIndexingWalRestoreTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodeWithIndexingWalRestoreTest.java new file mode 100644 index 0000000000000..385cf1906c644 --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodeWithIndexingWalRestoreTest.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.database; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import org.apache.ignite.IgniteBinary; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.binary.BinaryObject; +import org.apache.ignite.binary.BinaryObjectBuilder; +import org.apache.ignite.cache.QueryEntity; +import org.apache.ignite.cache.QueryIndex; +import org.apache.ignite.configuration.BinaryConfiguration; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.PersistentStoreConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.binary.BinaryMarshaller; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * Test verifies that binary metadata of values stored in cache and indexes upon these values + * is handled correctly on cluster restart when persistent store is enabled and compact footer is turned on. + */ +public class IgniteDbSingleNodeWithIndexingWalRestoreTest extends GridCommonAbstractTest { + /** */ + private static final String BINARY_TYPE_NAME = "BinaryPerson"; + + /** */ + private static final String BINARY_TYPE_FIELD_NAME = "binaryName"; + + /** */ + private static int ENTRIES_COUNT = 500; + + /** */ + private static class RegularPerson { + /** */ + private String regName; + + /** */ + public RegularPerson(String regName) { + this.regName = regName; + } + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(gridName); + + BinaryConfiguration binCfg = new BinaryConfiguration(); + binCfg.setCompactFooter(true); + + cfg.setBinaryConfiguration(binCfg); + + CacheConfiguration indexedCacheCfg = new CacheConfiguration(); + + indexedCacheCfg.setName("indexedCache"); + + List qryEntities = new ArrayList<>(); + + { + QueryEntity qryEntity = new QueryEntity(); + qryEntity.setKeyType(Integer.class.getName()); + qryEntity.setValueType(BINARY_TYPE_NAME); + + LinkedHashMap fields = new LinkedHashMap<>(); + fields.put(BINARY_TYPE_FIELD_NAME, String.class.getName()); + + qryEntity.setFields(fields); + + qryEntity.setIndexes(F.asList(new QueryIndex(BINARY_TYPE_FIELD_NAME))); + + qryEntities.add(qryEntity); + } + + { + QueryEntity qryEntity = new QueryEntity(); + qryEntity.setKeyType(Integer.class.getName()); + qryEntity.setValueType(RegularPerson.class.getName()); + + LinkedHashMap fields = new LinkedHashMap<>(); + fields.put("regName", String.class.getName()); + + qryEntity.setFields(fields); + + qryEntity.setIndexes(F.asList(new QueryIndex("regName"))); + + qryEntities.add(qryEntity); + } + + indexedCacheCfg.setQueryEntities(qryEntities); + + cfg.setCacheConfiguration(indexedCacheCfg); + + cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration()); + + cfg.setConsistentId(gridName); + + return cfg; + } + + /** + * Test for values without class created with BinaryObjectBuilder. + */ + public void testClasslessBinaryValuesRestored() throws Exception { + IgniteEx ig = startGrid(0); + + ig.active(true); + + GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) ig.context().cache().context().database(); + + dbMgr.enableCheckpoints(false).get(); + + IgniteCache cache = ig.cache("indexedCache").withKeepBinary(); + + IgniteBinary bin = ig.binary(); + + for (int i = 0; i < ENTRIES_COUNT; i++) { + BinaryObjectBuilder bldr = bin.builder(BINARY_TYPE_NAME); + + bldr.setField(BINARY_TYPE_FIELD_NAME, "Peter" + i); + + cache.put(i, bldr.build()); + } + + stopGrid(0, true); + + ig = startGrid(0); + + ig.active(true); + + cache = ig.cache("indexedCache").withKeepBinary(); + + for (int i = 0; i < ENTRIES_COUNT; i++) + assertEquals("Peter" + i, (((BinaryObject)cache.get(i)).field(BINARY_TYPE_FIELD_NAME))); + } + + /** + * Test for regular objects stored in cache with compactFooter=true setting + * (no metainformation to deserialize values is stored with values themselves). + */ + public void testRegularClassesRestored() throws Exception { + IgniteEx ig = startGrid(0); + + ig.active(true); + + GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) ig.context().cache().context().database(); + + dbMgr.enableCheckpoints(false).get(); + + IgniteCache cache = ig.cache("indexedCache"); + + for (int i = 0; i < ENTRIES_COUNT; i++) + cache.put(i, new RegularPerson("RegularPeter" + i)); + + stopGrid(0, true); + + ig = startGrid(0); + + ig.active(true); + + cache = ig.cache("indexedCache"); + + for (int i = 0; i < ENTRIES_COUNT; i++) + assertEquals("RegularPeter" + i, ((RegularPerson)cache.get(i)).regName); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false)); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false)); + } +} diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java index 15ea594c1d3c5..033e5a9751308 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java @@ -22,6 +22,7 @@ import org.apache.ignite.internal.processors.cache.IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest; import org.apache.ignite.internal.processors.database.IgniteDbMultiNodeWithIndexingPutGetTest; import org.apache.ignite.internal.processors.database.IgniteDbSingleNodeWithIndexingPutGetTest; +import org.apache.ignite.internal.processors.database.IgniteDbSingleNodeWithIndexingWalRestoreTest; import org.apache.ignite.internal.processors.database.IgnitePersistentStoreQueryWithMultipleClassesPerCacheTest; import org.apache.ignite.internal.processors.database.IgnitePersistentStoreSchemaLoadTest; @@ -36,6 +37,7 @@ public class IgnitePdsWithIndexingTestSuite extends TestSuite { public static TestSuite suite() throws Exception { TestSuite suite = new TestSuite("Ignite Db Memory Leaks With Indexing Test Suite"); + suite.addTestSuite(IgniteDbSingleNodeWithIndexingWalRestoreTest.class); suite.addTestSuite(IgniteDbSingleNodeWithIndexingPutGetTest.class); suite.addTestSuite(IgniteDbMultiNodeWithIndexingPutGetTest.class); suite.addTestSuite(IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest.class); From 3a3650fb2eadcd6a1186cc186fa5d471cd1e74d5 Mon Sep 17 00:00:00 2001 From: sboikov Date: Thu, 27 Jul 2017 13:57:11 +0300 Subject: [PATCH 044/547] Added related GridIoPolicy in IgniteThread. --- .../apache/ignite/internal/IgnitionEx.java | 34 ++++++++++++------ .../managers/communication/GridIoPolicy.java | 3 ++ .../service/GridServiceProcessor.java | 2 +- .../util/StripedCompositeReadWriteLock.java | 6 ++-- .../ignite/internal/util/StripedExecutor.java | 4 ++- .../apache/ignite/thread/IgniteThread.java | 35 +++++++++---------- .../ignite/thread/IgniteThreadFactory.java | 15 +++++--- .../thread/IgniteThreadPoolExecutor.java | 33 ++++++++++++++++- ...GridThreadPoolExecutorServiceSelfTest.java | 2 +- 9 files changed, 95 insertions(+), 39 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java index 1139ec60860d0..23baeb3781e62 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java @@ -64,6 +64,7 @@ import org.apache.ignite.configuration.MemoryConfiguration; import org.apache.ignite.configuration.TransactionConfiguration; import org.apache.ignite.internal.binary.BinaryMarshaller; +import org.apache.ignite.internal.managers.communication.GridIoPolicy; import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor; import org.apache.ignite.internal.processors.igfs.IgfsThreadFactory; import org.apache.ignite.internal.processors.igfs.IgfsUtils; @@ -1694,7 +1695,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException { cfg.getPublicThreadPoolSize(), cfg.getPublicThreadPoolSize(), DFLT_THREAD_KEEP_ALIVE_TIME, - new LinkedBlockingQueue()); + new LinkedBlockingQueue(), + GridIoPolicy.PUBLIC_POOL); execSvc.allowCoreThreadTimeOut(true); @@ -1706,7 +1708,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException { cfg.getServiceThreadPoolSize(), cfg.getServiceThreadPoolSize(), DFLT_THREAD_KEEP_ALIVE_TIME, - new LinkedBlockingQueue()); + new LinkedBlockingQueue(), + GridIoPolicy.SERVICE_POOL); svcExecSvc.allowCoreThreadTimeOut(true); @@ -1718,7 +1721,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException { cfg.getSystemThreadPoolSize(), cfg.getSystemThreadPoolSize(), DFLT_THREAD_KEEP_ALIVE_TIME, - new LinkedBlockingQueue()); + new LinkedBlockingQueue(), + GridIoPolicy.SYSTEM_POOL); sysExecSvc.allowCoreThreadTimeOut(true); @@ -1738,7 +1742,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException { cfg.getManagementThreadPoolSize(), cfg.getManagementThreadPoolSize(), DFLT_THREAD_KEEP_ALIVE_TIME, - new LinkedBlockingQueue()); + new LinkedBlockingQueue(), + GridIoPolicy.MANAGEMENT_POOL); mgmtExecSvc.allowCoreThreadTimeOut(true); @@ -1753,7 +1758,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException { cfg.getPeerClassLoadingThreadPoolSize(), cfg.getPeerClassLoadingThreadPoolSize(), DFLT_THREAD_KEEP_ALIVE_TIME, - new LinkedBlockingQueue()); + new LinkedBlockingQueue(), + GridIoPolicy.P2P_POOL); p2pExecSvc.allowCoreThreadTimeOut(true); @@ -1764,7 +1770,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException { cfg.getDataStreamerThreadPoolSize(), cfg.getDataStreamerThreadPoolSize(), DFLT_THREAD_KEEP_ALIVE_TIME, - new LinkedBlockingQueue()); + new LinkedBlockingQueue(), + GridIoPolicy.DATA_STREAMER_POOL); dataStreamerExecSvc.allowCoreThreadTimeOut(true); @@ -1811,7 +1818,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException { myCfg.getUtilityCacheThreadPoolSize(), myCfg.getUtilityCacheThreadPoolSize(), myCfg.getUtilityCacheKeepAliveTime(), - new LinkedBlockingQueue()); + new LinkedBlockingQueue(), + GridIoPolicy.UTILITY_CACHE_POOL); utilityCacheExecSvc.allowCoreThreadTimeOut(true); @@ -1821,7 +1829,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException { 1, 1, DFLT_THREAD_KEEP_ALIVE_TIME, - new LinkedBlockingQueue()); + new LinkedBlockingQueue(), + GridIoPolicy.AFFINITY_POOL); affExecSvc.allowCoreThreadTimeOut(true); @@ -1834,7 +1843,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException { cpus, cpus * 2, 3000L, - new LinkedBlockingQueue(1000) + new LinkedBlockingQueue(1000), + GridIoPolicy.IDX_POOL ); } @@ -1846,7 +1856,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException { cfg.getQueryThreadPoolSize(), cfg.getQueryThreadPoolSize(), DFLT_THREAD_KEEP_ALIVE_TIME, - new LinkedBlockingQueue()); + new LinkedBlockingQueue(), + GridIoPolicy.QUERY_POOL); qryExecSvc.allowCoreThreadTimeOut(true); @@ -1856,7 +1867,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException { 2, 2, DFLT_THREAD_KEEP_ALIVE_TIME, - new LinkedBlockingQueue()); + new LinkedBlockingQueue(), + GridIoPolicy.SCHEMA_POOL); schemaExecSvc.allowCoreThreadTimeOut(true); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoPolicy.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoPolicy.java index 13bc4c470603b..3f31f92088315 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoPolicy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoPolicy.java @@ -22,6 +22,9 @@ * message processing by the communication manager. */ public class GridIoPolicy { + /** */ + public static final byte UNDEFINED = -1; + /** Public execution pool. */ public static final byte PUBLIC_POOL = 0; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java index db632ec7b1cf8..46fcfea8ef509 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java @@ -153,7 +153,7 @@ public class GridServiceProcessor extends GridProcessorAdapter implements Ignite private volatile GridSpinBusyLock busyLock = new GridSpinBusyLock(); /** Thread factory. */ - private ThreadFactory threadFactory = new IgniteThreadFactory(ctx.igniteInstanceName()); + private ThreadFactory threadFactory = new IgniteThreadFactory(ctx.igniteInstanceName(), "service"); /** Thread local for service name. */ private ThreadLocal svcName = new ThreadLocal<>(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/StripedCompositeReadWriteLock.java b/modules/core/src/main/java/org/apache/ignite/internal/util/StripedCompositeReadWriteLock.java index e215663dbcc71..18ef06cdd5401 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/StripedCompositeReadWriteLock.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/StripedCompositeReadWriteLock.java @@ -67,8 +67,10 @@ public StripedCompositeReadWriteLock(int concurrencyLvl) { @NotNull @Override public Lock readLock() { int idx; - if (Thread.currentThread() instanceof IgniteThread) { - IgniteThread igniteThread = (IgniteThread)Thread.currentThread(); + Thread curThread = Thread.currentThread(); + + if (curThread instanceof IgniteThread) { + IgniteThread igniteThread = (IgniteThread)curThread; idx = igniteThread.compositeRwLockIndex(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java b/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java index 6c85b32c89734..6d5dc71335d62 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java @@ -35,6 +35,7 @@ import java.util.concurrent.locks.LockSupport; import org.apache.ignite.IgniteInterruptedException; import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.managers.communication.GridIoPolicy; import org.apache.ignite.internal.util.typedef.internal.A; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; @@ -439,7 +440,8 @@ void start() { poolName + "-stripe-" + idx, this, IgniteThread.GRP_IDX_UNASSIGNED, - idx); + idx, + GridIoPolicy.UNDEFINED); thread.start(); } diff --git a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThread.java b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThread.java index 6005ac9a17f8c..c814625d23fb3 100644 --- a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThread.java +++ b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThread.java @@ -18,6 +18,7 @@ package org.apache.ignite.thread; import java.util.concurrent.atomic.AtomicLong; +import org.apache.ignite.internal.managers.communication.GridIoPolicy; import org.apache.ignite.internal.util.typedef.internal.A; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.worker.GridWorker; @@ -52,13 +53,16 @@ public class IgniteThread extends Thread { /** */ private final int stripe; + /** */ + private final byte plc; + /** * Creates thread with given worker. * * @param worker Runnable to create thread with. */ public IgniteThread(GridWorker worker) { - this(DFLT_GRP, worker.igniteInstanceName(), worker.name(), worker, GRP_IDX_UNASSIGNED, -1); + this(worker.igniteInstanceName(), worker.name(), worker, GRP_IDX_UNASSIGNED, -1, GridIoPolicy.UNDEFINED); } /** @@ -69,41 +73,28 @@ public IgniteThread(GridWorker worker) { * @param r Runnable to execute. */ public IgniteThread(String igniteInstanceName, String threadName, Runnable r) { - this(igniteInstanceName, threadName, r, GRP_IDX_UNASSIGNED, -1); - } - - /** - * Creates grid thread with given name for a given Ignite instance. - * - * @param igniteInstanceName Name of the Ignite instance this thread is created for. - * @param threadName Name of thread. - * @param r Runnable to execute. - * @param grpIdx Index within a group. - * @param stripe Non-negative stripe number if this thread is striped pool thread. - */ - public IgniteThread(String igniteInstanceName, String threadName, Runnable r, int grpIdx, int stripe) { - this(DFLT_GRP, igniteInstanceName, threadName, r, grpIdx, stripe); + this(igniteInstanceName, threadName, r, GRP_IDX_UNASSIGNED, -1, GridIoPolicy.UNDEFINED); } /** * Creates grid thread with given name for a given Ignite instance with specified * thread group. * - * @param grp Thread group. * @param igniteInstanceName Name of the Ignite instance this thread is created for. * @param threadName Name of thread. * @param r Runnable to execute. * @param grpIdx Thread index within a group. * @param stripe Non-negative stripe number if this thread is striped pool thread. */ - public IgniteThread(ThreadGroup grp, String igniteInstanceName, String threadName, Runnable r, int grpIdx, int stripe) { - super(grp, r, createName(cntr.incrementAndGet(), threadName, igniteInstanceName)); + public IgniteThread(String igniteInstanceName, String threadName, Runnable r, int grpIdx, int stripe, byte plc) { + super(DFLT_GRP, r, createName(cntr.incrementAndGet(), threadName, igniteInstanceName)); A.ensure(grpIdx >= -1, "grpIdx >= -1"); this.igniteInstanceName = igniteInstanceName; this.compositeRwLockIdx = grpIdx; this.stripe = stripe; + this.plc = plc; } /** @@ -117,6 +108,14 @@ protected IgniteThread(String igniteInstanceName, ThreadGroup threadGrp, String this.igniteInstanceName = igniteInstanceName; this.compositeRwLockIdx = GRP_IDX_UNASSIGNED; this.stripe = -1; + this.plc = GridIoPolicy.UNDEFINED; + } + + /** + * @return Related {@link GridIoPolicy} for internal Ignite pools. + */ + public byte policy() { + return plc; } /** diff --git a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadFactory.java b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadFactory.java index d2f0b159aaf24..062c973ba1220 100644 --- a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadFactory.java +++ b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadFactory.java @@ -20,6 +20,7 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.ignite.internal.managers.communication.GridIoPolicy; import org.apache.ignite.internal.util.typedef.internal.S; import org.jetbrains.annotations.NotNull; @@ -37,14 +38,18 @@ public class IgniteThreadFactory implements ThreadFactory { /** Index generator for threads. */ private final AtomicInteger idxGen = new AtomicInteger(); + /** */ + private final byte plc; + /** * Constructs new thread factory for given grid. All threads will belong * to the same default thread group. * * @param igniteInstanceName Ignite instance name. + * @param threadName Thread name. */ - public IgniteThreadFactory(String igniteInstanceName) { - this(igniteInstanceName, "ignite"); + public IgniteThreadFactory(String igniteInstanceName, String threadName) { + this(igniteInstanceName, threadName, GridIoPolicy.UNDEFINED); } /** @@ -53,15 +58,17 @@ public IgniteThreadFactory(String igniteInstanceName) { * * @param igniteInstanceName Ignite instance name. * @param threadName Thread name. + * @param plc {@link GridIoPolicy} for thread pool. */ - public IgniteThreadFactory(String igniteInstanceName, String threadName) { + public IgniteThreadFactory(String igniteInstanceName, String threadName, byte plc) { this.igniteInstanceName = igniteInstanceName; this.threadName = threadName; + this.plc = plc; } /** {@inheritDoc} */ @Override public Thread newThread(@NotNull Runnable r) { - return new IgniteThread(igniteInstanceName, threadName, r, idxGen.incrementAndGet(), -1); + return new IgniteThread(igniteInstanceName, threadName, r, idxGen.incrementAndGet(), -1, plc); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadPoolExecutor.java b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadPoolExecutor.java index 639ef9438a669..83c64c3363000 100644 --- a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadPoolExecutor.java +++ b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadPoolExecutor.java @@ -22,6 +22,7 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import org.apache.ignite.internal.managers.communication.GridIoPolicy; /** * An {@link ExecutorService} that executes submitted tasks using pooled grid threads. @@ -46,13 +47,43 @@ public IgniteThreadPoolExecutor( int maxPoolSize, long keepAliveTime, BlockingQueue workQ) { + this(threadNamePrefix, + igniteInstanceName, + corePoolSize, + maxPoolSize, + keepAliveTime, + workQ, + GridIoPolicy.UNDEFINED); + } + + /** + * Creates a new service with the given initial parameters. + * + * @param threadNamePrefix Will be added at the beginning of all created threads. + * @param igniteInstanceName Must be the name of the grid. + * @param corePoolSize The number of threads to keep in the pool, even if they are idle. + * @param maxPoolSize The maximum number of threads to allow in the pool. + * @param keepAliveTime When the number of threads is greater than the core, this is the maximum time + * that excess idle threads will wait for new tasks before terminating. + * @param workQ The queue to use for holding tasks before they are executed. This queue will hold only + * runnable tasks submitted by the {@link #execute(Runnable)} method. + * @param plc {@link GridIoPolicy} for thread pool. + */ + public IgniteThreadPoolExecutor( + String threadNamePrefix, + String igniteInstanceName, + int corePoolSize, + int maxPoolSize, + long keepAliveTime, + BlockingQueue workQ, + byte plc) { super( corePoolSize, maxPoolSize, keepAliveTime, TimeUnit.MILLISECONDS, workQ, - new IgniteThreadFactory(igniteInstanceName, threadNamePrefix) + new IgniteThreadFactory(igniteInstanceName, threadNamePrefix, plc) ); } diff --git a/modules/core/src/test/java/org/apache/ignite/thread/GridThreadPoolExecutorServiceSelfTest.java b/modules/core/src/test/java/org/apache/ignite/thread/GridThreadPoolExecutorServiceSelfTest.java index 3948f6a620bc6..dce6328775ef8 100644 --- a/modules/core/src/test/java/org/apache/ignite/thread/GridThreadPoolExecutorServiceSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/thread/GridThreadPoolExecutorServiceSelfTest.java @@ -64,7 +64,7 @@ public void testSingleThreadExecutor() throws Exception { * @throws Exception If failed. */ public void testSingleGridThreadExecutor() throws Exception { - ExecutorService exec = Executors.newSingleThreadExecutor(new IgniteThreadFactory("gridName")); + ExecutorService exec = Executors.newSingleThreadExecutor(new IgniteThreadFactory("gridName", "testThread")); exec.submit(new InterruptingRunnable()).get(); From eb9d06d9ff89388318cc3a857276c154675bc6f1 Mon Sep 17 00:00:00 2001 From: Dmitry Pavlov Date: Thu, 27 Jul 2017 14:51:25 +0300 Subject: [PATCH 045/547] ignite-5682 Added stale version check for GridDhtPartFullMessage not related to exchange. --- .../cache/CacheAffinitySharedManager.java | 5 +- .../processors/cache/GridCacheIoManager.java | 2 +- .../GridCachePartitionExchangeManager.java | 44 +++++++++----- .../dht/GridClientPartitionTopology.java | 12 +++- .../dht/GridDhtPartitionTopology.java | 9 ++- .../dht/GridDhtPartitionTopologyImpl.java | 34 ++++++----- .../preloader/GridDhtPartitionExchangeId.java | 2 +- .../GridDhtPartitionsExchangeFuture.java | 36 ++++++++---- .../GridDhtPartitionsFullMessage.java | 4 +- ...ngDelayedPartitionMapExchangeSelfTest.java | 58 +++++++++++++++---- .../junits/common/GridCommonAbstractTest.java | 6 +- 11 files changed, 151 insertions(+), 61 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java index 0f46a906cfbf6..5a7f634b7a5a7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java @@ -442,7 +442,8 @@ void onCacheGroupCreated(CacheGroupContext grp) { grp.topology().update(topVer, clientTop.partitionMap(true), clientTop.updateCounters(false), - Collections.emptySet()); + Collections.emptySet(), + null); } grpHolder = new CacheGroupHolder1(grp, grpHolder.affinity()); @@ -504,7 +505,7 @@ else if (!fetchFuts.containsKey(grp.groupId())) { grp.topology().updateTopologyVersion(topFut, discoCache, -1, false); - grp.topology().update(topVer, partMap, null, Collections.emptySet()); + grp.topology().update(topVer, partMap, null, Collections.emptySet(), null); topFut.validate(grp, discoCache.allNodes()); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java index 9f1873e01671a..981c6e2f1defa 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java @@ -533,7 +533,7 @@ public void writeUnlock() { } /** - * @param nodeId Node ID. + * @param nodeId Sender Node ID. * @param cacheMsg Cache message. * @param c Handler closure. * @param plc Message policy. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java index d4fe93f8567b3..6a7258fb08d95 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java @@ -129,7 +129,7 @@ public class GridCachePartitionExchangeManager extends GridCacheSharedMana private static final int EXCHANGE_HISTORY_SIZE = IgniteSystemProperties.getInteger(IgniteSystemProperties.IGNITE_EXCHANGE_HISTORY_SIZE, 1_000); - /** Atomic reference for pending timeout object. */ + /** Atomic reference for pending partition resend timeout object. */ private AtomicReference pendingResend = new AtomicReference<>(); /** Partition resend timeout after eviction. */ @@ -150,7 +150,7 @@ public class GridCachePartitionExchangeManager extends GridCacheSharedMana private final ConcurrentMap clientTops = new ConcurrentHashMap8<>(); /** */ - private volatile GridDhtPartitionsExchangeFuture lastInitializedFut; + @Nullable private volatile GridDhtPartitionsExchangeFuture lastInitializedFut; /** */ private final AtomicReference lastFinishedFut = new AtomicReference<>(); @@ -877,6 +877,8 @@ public void scheduleResendPartitions() { /** * Partition refresh callback. + * For coordinator causes {@link GridDhtPartitionsFullMessage FullMessages} send, + * for non coordinator - {@link GridDhtPartitionsSingleMessage SingleMessages} send */ private void refreshPartitions() { ClusterNode oldest = cctx.discovery().oldestAliveCacheServerNode(AffinityTopologyVersion.NONE); @@ -914,7 +916,7 @@ private void refreshPartitions() { if (log.isDebugEnabled()) log.debug("Refreshing partitions from oldest node: " + cctx.localNodeId()); - sendAllPartitions(rmts); + sendAllPartitions(rmts, rmtTopVer); } else { if (log.isDebugEnabled()) @@ -927,10 +929,14 @@ private void refreshPartitions() { /** * @param nodes Nodes. + * @param msgTopVer Topology version. Will be added to full message. */ - private void sendAllPartitions(Collection nodes) { + private void sendAllPartitions(Collection nodes, + AffinityTopologyVersion msgTopVer) { GridDhtPartitionsFullMessage m = createPartitionsFullMessage(true, null, null, null, null); + m.topologyVersion(msgTopVer); + if (log.isDebugEnabled()) log.debug("Sending all partitions [nodeIds=" + U.nodeIds(nodes) + ", msg=" + m + ']'); @@ -956,6 +962,8 @@ private void sendAllPartitions(Collection nodes) { * finishUnmarshall methods are called). * @param exchId Non-null exchange ID if message is created for exchange. * @param lastVer Last version. + * @param partHistSuppliers + * @param partsToReload * @return Message. */ public GridDhtPartitionsFullMessage createPartitionsFullMessage( @@ -1064,8 +1072,8 @@ private void addFullPartitionsMap(GridDhtPartitionsFullMessage m, } /** - * @param node Node. - * @param id ID. + * @param node Destination cluster node. + * @param id Exchange ID. */ private void sendLocalPartitions(ClusterNode node, @Nullable GridDhtPartitionExchangeId id) { GridDhtPartitionsSingleMessage m = createPartitionsSingleMessage(node, @@ -1091,7 +1099,7 @@ private void sendLocalPartitions(ClusterNode node, @Nullable GridDhtPartitionExc /** * @param targetNode Target node. - * @param exchangeId ID. + * @param exchangeId Exchange ID. * @param clientOnlyExchange Client exchange flag. * @param sndCounters {@code True} if need send partition update counters. * @return Message. @@ -1297,7 +1305,7 @@ private boolean addFuture(GridDhtPartitionsExchangeFuture fut) { } /** - * @param node Node. + * @param node Sender cluster node. * @param msg Message. */ private void processFullPartitionUpdate(ClusterNode node, GridDhtPartitionsFullMessage msg) { @@ -1323,8 +1331,13 @@ private void processFullPartitionUpdate(ClusterNode node, GridDhtPartitionsFullM else if (!grp.isLocal()) top = grp.topology(); - if (top != null) - updated |= top.update(null, entry.getValue(), null, msg.partsToReload(cctx.localNodeId(), grpId)); + if (top != null) { + updated |= top.update(null, + entry.getValue(), + null, + msg.partsToReload(cctx.localNodeId(), grpId), + msg.topologyVersion()); + } } if (!cctx.kernalContext().clientNode() && updated) @@ -1352,7 +1365,7 @@ else if (!grp.isLocal()) } /** - * @param node Node ID. + * @param node Sender cluster node. * @param msg Message. */ private void processSinglePartitionUpdate(final ClusterNode node, final GridDhtPartitionsSingleMessage msg) { @@ -1418,7 +1431,7 @@ else if (!grp.isLocal()) } /** - * @param node Node ID. + * @param node Sender cluster node. * @param msg Message. */ private void processSinglePartitionRequest(ClusterNode node, GridDhtPartitionsSingleRequest msg) { @@ -2250,7 +2263,10 @@ private abstract class MessageHandler implements IgniteBiInClosure { /** */ private static final long serialVersionUID = 0L; - /** {@inheritDoc} */ + /** + * @param nodeId Sender node ID. + * @param msg Message. + */ @Override public void apply(UUID nodeId, M msg) { ClusterNode node = cctx.node(nodeId); @@ -2268,7 +2284,7 @@ private abstract class MessageHandler implements IgniteBiInClosure { } /** - * @param node Node. + * @param node Sender cluster node. * @param msg Message. */ protected abstract void onMessage(ClusterNode node, M msg); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java index 7343dba714672..4b9826e8f14d9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java @@ -594,8 +594,8 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD @Nullable AffinityTopologyVersion exchangeVer, GridDhtPartitionFullMap partMap, Map> cntrMap, - Set partsToReload - ) { + Set partsToReload, + @Nullable AffinityTopologyVersion msgTopVer) { if (log.isDebugEnabled()) log.debug("Updating full partition map [exchVer=" + exchangeVer + ", parts=" + fullMapString() + ']'); @@ -610,6 +610,14 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD return false; } + if (msgTopVer != null && lastExchangeVer != null && lastExchangeVer.compareTo(msgTopVer) > 0) { + if (log.isDebugEnabled()) + log.debug("Stale topology version for full partition map update message (will ignore) " + + "[lastExchId=" + lastExchangeVer + ", topVersion=" + msgTopVer + ']'); + + return false; + } + boolean fullMapUpdated = (node2part == null); if (node2part != null) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java index d9e04a6da4a46..81d92e0a72588 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java @@ -243,16 +243,21 @@ public void beforeExchange(GridDhtPartitionsExchangeFuture exchFut, boolean affR public void onRemoved(GridDhtCacheEntry e); /** - * @param exchangeVer Exchange version. + * @param exchangeVer Topology version from exchange. Value should be greater than previously passed. Null value + * means full map received is not related to exchange * @param partMap Update partition map. * @param cntrMap Partition update counters. + * @param partsToReload + * @param msgTopVer Topology version from incoming message. This value is not null only for case message is not + * related to exchange. Value should be not less than previous 'Topology version from exchange'. * @return {@code True} if local state was changed. */ public boolean update( @Nullable AffinityTopologyVersion exchangeVer, GridDhtPartitionFullMap partMap, @Nullable Map> cntrMap, - Set partsToReload); + Set partsToReload, + @Nullable AffinityTopologyVersion msgTopVer); /** * @param exchId Exchange ID. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java index 842501e7be4dd..a8e13a0f14469 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java @@ -440,7 +440,7 @@ else if (localNode(p, aff)) if (exchId.isLeft() && exchFut.serverNodeDiscoveryEvent()) removeNode(exchId.nodeId()); - + ClusterNode oldest = discoCache.oldestAliveServerNodeWithCache(); if (log.isDebugEnabled()) { @@ -1099,9 +1099,9 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD @Override public boolean update( @Nullable AffinityTopologyVersion exchangeVer, GridDhtPartitionFullMap partMap, - @Nullable Map> cntrMap, - Set partsToReload - ) { + @Nullable Map> incomeCntrMap, + Set partsToReload, + @Nullable AffinityTopologyVersion msgTopVer) { if (log.isDebugEnabled()) log.debug("Updating full partition map [exchVer=" + exchangeVer + ", parts=" + fullMapString() + ']'); @@ -1113,12 +1113,12 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD if (stopping) return false; - if (cntrMap != null) { + if (incomeCntrMap != null) { // update local map partition counters - for (Map.Entry> e : cntrMap.entrySet()) { - T2 cntr = this.cntrMap.get(e.getKey()); + for (Map.Entry> e : incomeCntrMap.entrySet()) { + T2 existCntr = this.cntrMap.get(e.getKey()); - if (cntr == null || cntr.get2() < e.getValue().get2()) + if (existCntr == null || existCntr.get2() < e.getValue().get2()) this.cntrMap.put(e.getKey(), e.getValue()); } @@ -1129,7 +1129,7 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD if (part == null) continue; - T2 cntr = cntrMap.get(part.id()); + T2 cntr = incomeCntrMap.get(part.id()); if (cntr != null) part.updateCounter(cntr.get2()); @@ -1144,6 +1144,14 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD return false; } + if (msgTopVer != null && lastExchangeVer != null && lastExchangeVer.compareTo(msgTopVer) > 0) { + if (log.isDebugEnabled()) + log.debug("Stale version for full partition map update message (will ignore) [lastExch=" + + lastExchangeVer + ", topVersion=" + msgTopVer + ']'); + + return false; + } + boolean fullMapUpdated = (node2part == null); if (node2part != null) { @@ -1244,8 +1252,8 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD assert locPart != null; - if (cntrMap != null) { - T2 cntr = cntrMap.get(p); + if (incomeCntrMap != null) { + T2 cntr = incomeCntrMap.get(p); if (cntr != null && cntr.get2() > locPart.updateCounter()) locPart.updateCounter(cntr.get2()); @@ -1271,8 +1279,8 @@ else if (state == MOVING) { changed = true; } - if (cntrMap != null) { - T2 cntr = cntrMap.get(p); + if (incomeCntrMap != null) { + T2 cntr = incomeCntrMap.get(p); if (cntr != null && cntr.get2() > locPart.updateCounter()) locPart.updateCounter(cntr.get2()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionExchangeId.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionExchangeId.java index 0a4941533cedc..1a4dabfc30d6a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionExchangeId.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionExchangeId.java @@ -51,7 +51,7 @@ public class GridDhtPartitionExchangeId implements Message, Comparable remaining = new HashSet<>(); - /** */ + /** Guarded by this */ @GridToStringExclude private int pendingSingleUpdates; @@ -154,7 +154,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte @GridToStringExclude private final CountDownLatch evtLatch = new CountDownLatch(1); - /** */ + /** Exchange future init method completes this future. */ private GridFutureAdapter initFut; /** */ @@ -196,7 +196,10 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte /** Init timestamp. Used to track the amount of time spent to complete the future. */ private long initTs; - /** */ + /** + * Centralized affinity assignment required. Activated for node left of failed. For this mode crd will send full + * partitions maps to nodes using discovery (ring) instead of communication. + */ private boolean centralizedAff; /** Change global state exception. */ @@ -613,7 +616,8 @@ private void updateTopologies(boolean crd) throws IgniteCheckedException { top.update(topologyVersion(), clientTop.partitionMap(true), clientTop.updateCounters(false), - Collections.emptySet()); + Collections.emptySet(), + null); } } @@ -1092,7 +1096,7 @@ private boolean cacheStopping(int cacheId) { } /** - * @param node Node. + * @param node Target Node. * @throws IgniteCheckedException If failed. */ private void sendLocalPartitions(ClusterNode node) throws IgniteCheckedException { @@ -1191,7 +1195,7 @@ private void sendAllPartitions(Collection nodes) throws IgniteCheck } /** - * @param oldestNode Oldest node. + * @param oldestNode Oldest node. Target node to send message to. */ private void sendPartitions(ClusterNode oldestNode) { try { @@ -1368,6 +1372,9 @@ private void updateLastVersion(GridCacheVersion ver) { } /** + * Processing of received single message. Actual processing in future may be delayed if init method was not + * completed, see {@link #initDone()} + * * @param node Sender node. * @param msg Single partition info. */ @@ -1409,11 +1416,14 @@ public void onReceive(final ClusterNode node, final GridDhtPartitionsSingleMessa } /** + * Note this method performs heavy updatePartitionSingleMap operation, this operation is moved out from the + * synchronized block. Only count of such updates {@link #pendingSingleUpdates} is managed under critical section. + * * @param node Sender node. - * @param msg Message. + * @param msg Partition single message. */ private void processMessage(ClusterNode node, GridDhtPartitionsSingleMessage msg) { - boolean allReceived = false; + boolean allReceived = false; // Received all expected messages. boolean updateSingleMap = false; synchronized (this) { @@ -1895,7 +1905,7 @@ public void onReceive(final ClusterNode node, final GridDhtPartitionsFullMessage /** * @param node Sender node. - * @param msg Message. + * @param msg Message with full partition info. */ private void processMessage(ClusterNode node, GridDhtPartitionsFullMessage msg) { assert exchId.equals(msg.exchangeId()) : msg; @@ -1953,7 +1963,8 @@ private void updatePartitionFullMap(GridDhtPartitionsFullMessage msg) { grp.topology().update(topologyVersion(), entry.getValue(), cntrMap, - msg.partsToReload(cctx.localNodeId(), grpId)); + msg.partsToReload(cctx.localNodeId(), grpId), + null); } else { ClusterNode oldest = cctx.discovery().oldestAliveCacheServerNode(AffinityTopologyVersion.NONE); @@ -1962,7 +1973,8 @@ private void updatePartitionFullMap(GridDhtPartitionsFullMessage msg) { cctx.exchange().clientTopology(grpId, this).update(topologyVersion(), entry.getValue(), cntrMap, - Collections.emptySet()); + Collections.emptySet(), + null); } } } @@ -2054,7 +2066,7 @@ private void onDiscoveryEvent(IgniteRunnable c) { } /** - * + * Moves exchange future to state 'init done' using {@link #initFut}. */ private void initDone() { while (!isDone()) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java index 75609b8ec49d0..acc4dbe1d455c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java @@ -109,7 +109,9 @@ public GridDhtPartitionsFullMessage() { /** * @param id Exchange ID. * @param lastVer Last version. - * @param topVer Topology version. + * @param topVer Topology version. For messages not related to exchange may be {@link AffinityTopologyVersion#NONE}. + * @param partHistSuppliers + * @param partsToReload */ public GridDhtPartitionsFullMessage(@Nullable GridDhtPartitionExchangeId id, @Nullable GridCacheVersion lastVer, diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRabalancingDelayedPartitionMapExchangeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRabalancingDelayedPartitionMapExchangeSelfTest.java index dc141db2e6032..f307b6a60ae3f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRabalancingDelayedPartitionMapExchangeSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRabalancingDelayedPartitionMapExchangeSelfTest.java @@ -18,14 +18,17 @@ package org.apache.ignite.internal.processors.cache.distributed.rebalancing; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.ignite.IgniteException; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.CacheRebalanceMode; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.internal.IgniteKernal; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.managers.communication.GridIoMessage; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsAbstractMessage; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteInClosure; @@ -45,12 +48,19 @@ public class GridCacheRabalancingDelayedPartitionMapExchangeSelfTest extends Gri /** */ protected static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); - /** */ + /** Map of destination node ID to runnable with logic for real message sending. + * To apply real message sending use run method */ private final ConcurrentHashMap8 rs = new ConcurrentHashMap8<>(); - /** */ + /** + * Flag to redirect {@link GridDhtPartitionsFullMessage}s from real communication channel to {@link #rs} map. + * Applied only to messages not related to particular exchange + */ private volatile boolean record = false; + /** */ + private AtomicBoolean replay = new AtomicBoolean(); + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration iCfg = super.getConfiguration(igniteInstanceName); @@ -74,13 +84,26 @@ public class DelayableCommunicationSpi extends TcpCommunicationSpi { final IgniteInClosure ackC) throws IgniteSpiException { final Object msg0 = ((GridIoMessage)msg).message(); + if (log.isDebugEnabled()) + log.debug("Message [thread=" + Thread.currentThread().getName() + ", msg=" + msg0 + ']'); + if (msg0 instanceof GridDhtPartitionsFullMessage && record && - ((GridDhtPartitionsFullMessage)msg0).exchangeId() == null) { - rs.putIfAbsent(node.id(), new Runnable() { + ((GridDhtPartitionsAbstractMessage)msg0).exchangeId() == null) { + if (log.isDebugEnabled()) + log.debug("Record message [toNode=" + node.id() + ", msg=" + msg + "]"); + + assert !replay.get() : "Record of message is not allowed after replay"; + + Runnable prevValue = rs.putIfAbsent(node.id(), new Runnable() { @Override public void run() { + if (log.isDebugEnabled()) + log.debug("Replay: " + msg); + DelayableCommunicationSpi.super.sendMessage(node, msg, ackC); } }); + + assert prevValue == null : "Duplicate message registered to [" + node.id() + "]"; } else try { @@ -94,10 +117,10 @@ public class DelayableCommunicationSpi extends TcpCommunicationSpi { } /** - * @throws Exception e. + * @throws Exception e if failed. */ public void test() throws Exception { - IgniteKernal ignite = (IgniteKernal)startGrid(0); + IgniteEx ignite = startGrid(0); CacheConfiguration cfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME); @@ -144,10 +167,7 @@ record = true; awaitPartitionMapExchange(); - for (Runnable r : rs.values()) - r.run(); - - U.sleep(10000); // Enough time to process delayed GridDhtPartitionsFullMessages. + replayMessages(); stopGrid(3); // Forces exchange at all nodes and cause assertion failure in case obsolete partition map accepted. @@ -167,6 +187,22 @@ record = true; assert grid(2).context().cache().context().exchange().readyAffinityVersion().topologyVersion() > topVer2; } + /** + * Replays all saved messages from map, actual sent is performed. + * + * @throws IgniteInterruptedCheckedException If interrupted. + */ + private void replayMessages() throws IgniteInterruptedCheckedException { + record = false; + + for (Runnable r : rs.values()) + r.run(); // Causes real messages sending. + + assertTrue(replay.compareAndSet(false, true)); + + U.sleep(10000); // Enough time to process delayed GridDhtPartitionsFullMessages. + } + /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { super.afterTest(); diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java index dc7e89d6815cb..c2cf41c7cca95 100755 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java @@ -520,7 +520,8 @@ protected void awaitPartitionMapExchange() throws InterruptedException { /** * @param waitEvicts If {@code true} will wait for evictions finished. * @param waitNode2PartUpdate If {@code true} will wait for nodes node2part info update finished. - * @param nodes Optional nodes. + * @param nodes Optional nodes. If {@code null} method will wait for all nodes, for non null collection nodes will + * be filtered * @throws InterruptedException If interrupted. */ @SuppressWarnings("BusyWait") @@ -542,7 +543,8 @@ protected long getPartitionMapExchangeTimeout() { /** * @param waitEvicts If {@code true} will wait for evictions finished. * @param waitNode2PartUpdate If {@code true} will wait for nodes node2part info update finished. - * @param nodes Optional nodes. + * @param nodes Optional nodes. If {@code null} method will wait for all nodes, for non null collection nodes will + * be filtered * @param printPartState If {@code true} will print partition state if evictions not happened. * @throws InterruptedException If interrupted. */ From a07d7b91d148df8dfd7c4bbad2a63f8dea97b036 Mon Sep 17 00:00:00 2001 From: Alexandr Kuramshin Date: Thu, 27 Jul 2017 15:04:42 +0300 Subject: [PATCH 046/547] IGNITE-4767: rollback exception hides the origin exception (e.g. commit): Suppressing or logging rollback exceptions instead of hiding the origin exception Fixes #1599 --- .../processors/cache/GridCacheAdapter.java | 16 ++++-- .../dht/GridDhtTransactionalCacheAdapter.java | 7 ++- .../dht/GridDhtTxPrepareFuture.java | 24 +++++++-- .../cache/transactions/IgniteTxHandler.java | 51 +++++++++++++++---- .../cache/transactions/IgniteTxManager.java | 7 ++- .../ignite/stream/flume/IgniteSink.java | 7 ++- 6 files changed, 91 insertions(+), 21 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java index 9213be3cf5e4b..4ba4e48265ebd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java @@ -4072,7 +4072,7 @@ public void awaitLastFut() { catch (IgniteCheckedException | AssertionError | RuntimeException e1) { U.error(log, "Failed to rollback transaction (cache may contain stale locks): " + tx, e1); - U.addLastCause(e, e1, log); + e.addSuppressed(e1); } } @@ -4205,7 +4205,12 @@ protected IgniteInternalFuture asyncOp( throw e; } catch (IgniteCheckedException e1) { - tx0.rollbackNearTxLocalAsync(); + try { + tx0.rollbackNearTxLocalAsync(); + } + catch (Throwable e2) { + e1.addSuppressed(e2); + } throw e1; } @@ -4231,7 +4236,12 @@ protected IgniteInternalFuture asyncOp( throw e; } catch (IgniteCheckedException e1) { - tx0.rollbackNearTxLocalAsync(); + try { + tx0.rollbackNearTxLocalAsync(); + } + catch (Throwable e2) { + e1.addSuppressed(e2); + } throw e1; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java index 73942ffe83243..063986f90e462 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java @@ -1338,7 +1338,12 @@ private void sendLockReply( ", res=" + res + ']', e); if (tx != null) - tx.rollbackDhtLocalAsync(); + try { + tx.rollbackDhtLocalAsync(); + } + catch (Throwable e1) { + e.addSuppressed(e1); + } // Convert to closure exception as this method is only called form closures. throw new GridClosureException(e); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java index 6ed47811849cd..a31c540756b46 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java @@ -736,16 +736,27 @@ private boolean mapIfLocked() { tx.systemInvalidate(true); - fut = tx.rollbackAsync(); + try { + fut = tx.rollbackAsync(); - fut.listen(resClo); + fut.listen(resClo); + } + catch (Throwable e1) { + e.addSuppressed(e1); + } throw e; } } else if (!cctx.kernalContext().isStopping()) - fut = tx.rollbackAsync(); + try { + fut = tx.rollbackAsync(); + } + catch (Throwable e) { + err.addSuppressed(e); + fut = null; + } if (fut != null) fut.listen(resClo); @@ -1199,7 +1210,12 @@ private void prepare0() { if (err0 != null) { ERR_UPD.compareAndSet(this, null, err0); - tx.rollbackAsync(); + try { + tx.rollbackAsync(); + } + catch (Throwable e) { + err0.addSuppressed(e); + } final GridNearTxPrepareResponse res = createPrepareResponse(err); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java index c473bfe3f1324..b7ff319ea1f20 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java @@ -863,14 +863,17 @@ private IgniteInternalFuture finishDhtLocal(UUID nodeId, U.error(log, "Failed completing transaction [commit=" + req.commit() + ", tx=" + tx + ']', e); - IgniteInternalFuture res; + IgniteInternalFuture res = null; - IgniteInternalFuture rollbackFut = tx.rollbackDhtLocalAsync(); - - // Only for error logging. - rollbackFut.listen(CU.errorLogger(log)); + try { + res = tx.rollbackDhtLocalAsync(); - res = rollbackFut; + // Only for error logging. + res.listen(CU.errorLogger(log)); + } + catch (Throwable e1) { + e.addSuppressed(e1); + } if (e instanceof Error) throw (Error)e; @@ -906,7 +909,12 @@ public IgniteInternalFuture finishColocatedLocal(boolean commi throw e; if (tx != null) - return tx.rollbackNearTxLocalAsync(); + try { + return tx.rollbackNearTxLocalAsync(); + } + catch (Throwable e1) { + e.addSuppressed(e1); + } return new GridFinishedFuture<>(e); } @@ -1000,7 +1008,12 @@ else if (e instanceof IgniteTxHeuristicCheckedException) { U.error(log, "Failed to process prepare request: " + req, e); if (nearTx != null) - nearTx.rollbackRemoteTx(); + try { + nearTx.rollbackRemoteTx(); + } + catch (Throwable e1) { + e.addSuppressed(e1); + } res = new GridDhtTxPrepareResponse( req.partition(), @@ -1261,7 +1274,13 @@ protected void finish( tx.invalidate(true); tx.systemInvalidate(true); - tx.rollbackRemoteTx(); + try { + tx.rollbackRemoteTx(); + } + catch (Throwable e1) { + e.addSuppressed(e1); + U.error(log, "Failed to automatically rollback transaction: " + tx, e1); + } if (e instanceof Error) throw (Error)e; @@ -1307,10 +1326,20 @@ private void sendReply(UUID nodeId, } if (nearTx != null) - nearTx.rollbackRemoteTx(); + try { + nearTx.rollbackRemoteTx(); + } + catch (Throwable e1) { + e.addSuppressed(e1); + } if (dhtTx != null) - dhtTx.rollbackRemoteTx(); + try { + dhtTx.rollbackRemoteTx(); + } + catch (Throwable e1) { + e.addSuppressed(e1); + } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java index 26a4a91201f19..7d612eceb8be3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java @@ -2422,7 +2422,12 @@ private CommitListener(IgniteInternalTx tx) { log.debug("Optimistic failure while committing prepared transaction (will rollback): " + tx); - tx.rollbackAsync(); + try { + tx.rollbackAsync(); + } + catch (Throwable e) { + U.error(log, "Failed to automatically rollback transaction: " + tx, e); + } } catch (IgniteCheckedException e) { U.error(log, "Failed to commit transaction during failover: " + tx, e); diff --git a/modules/flume/src/main/java/org/apache/ignite/stream/flume/IgniteSink.java b/modules/flume/src/main/java/org/apache/ignite/stream/flume/IgniteSink.java index 083e8333796f8..64559ef5a3d25 100644 --- a/modules/flume/src/main/java/org/apache/ignite/stream/flume/IgniteSink.java +++ b/modules/flume/src/main/java/org/apache/ignite/stream/flume/IgniteSink.java @@ -178,7 +178,12 @@ public IgniteSink() { catch (Exception e) { log.error("Failed to process events", e); - transaction.rollback(); + try { + transaction.rollback(); + } + catch (Throwable e1) { + e.addSuppressed(e1); + } throw new EventDeliveryException(e); } From e9a0d694867467400e295fe71dba4244d8b54ecd Mon Sep 17 00:00:00 2001 From: sboikov Date: Thu, 27 Jul 2017 15:25:06 +0300 Subject: [PATCH 047/547] Minor (added more clear method IgniteTxHandler.prepareNearTxLocal). --- ...OptimisticSerializableTxPrepareFuture.java | 2 +- .../GridNearOptimisticTxPrepareFuture.java | 2 +- .../GridNearPessimisticTxPrepareFuture.java | 2 +- .../cache/transactions/IgniteTxHandler.java | 59 ++++++++++--------- 4 files changed, 35 insertions(+), 30 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java index 72ddc67e0a291..d7b47ded16e4d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java @@ -574,7 +574,7 @@ private void prepareLocal(GridNearTxPrepareRequest req, final MiniFuture fut, final boolean nearEntries) { IgniteInternalFuture prepFut = nearEntries ? - cctx.tm().txHandler().prepareNearTx(cctx.localNodeId(), req, true) : + cctx.tm().txHandler().prepareNearTxLocal(req) : cctx.tm().txHandler().prepareColocatedTx(tx, req); prepFut.listen(new CI1>() { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java index edddf7dd78a6b..df0d28e59918d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java @@ -564,7 +564,7 @@ private void proceedPrepare(GridDistributedTxMapping m, @Nullable final Queue prepFut = - m.hasNearCacheEntries() ? cctx.tm().txHandler().prepareNearTx(n.id(), req, true) + m.hasNearCacheEntries() ? cctx.tm().txHandler().prepareNearTxLocal(req) : cctx.tm().txHandler().prepareColocatedTx(tx, req); prepFut.listen(new CI1>() { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java index e934319c91ebd..2bcdade5cb456 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java @@ -237,7 +237,7 @@ private void prepareLocal(GridNearTxPrepareRequest req, add(fut); IgniteInternalFuture prepFut = nearEntries ? - cctx.tm().txHandler().prepareNearTx(cctx.localNodeId(), req, true) : + cctx.tm().txHandler().prepareNearTxLocal(req) : cctx.tm().txHandler().prepareColocatedTx(tx, req); prepFut.listen(new CI1>() { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java index 34a9fc1c4f715..2ed8ed7e6606f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java @@ -118,7 +118,19 @@ private void processNearTxPrepareRequest(final UUID nearNodeId, GridNearTxPrepar ", node=" + nearNodeId + ']'); } - IgniteInternalFuture fut = prepareNearTx(nearNodeId, req, false); + ClusterNode nearNode = ctx.node(nearNodeId); + + if (nearNode == null) { + if (txPrepareMsgLog.isDebugEnabled()) { + txPrepareMsgLog.debug("Received near prepare from node that left grid (will ignore) [" + + "txId=" + req.version() + + ", node=" + nearNodeId + ']'); + } + + return; + } + + IgniteInternalFuture fut = prepareNearTx(nearNode, req); assert req.txState() != null || fut == null || fut.error() != null || (ctx.tm().tx(req.version()) == null && ctx.tm().nearTx(req.version()) == null); @@ -278,32 +290,25 @@ private IgniteTxEntry unmarshal(@Nullable Collection entries) thr } /** - * @param nearNodeId Near node ID that initiated transaction. - * @param req Near prepare request. - * @param locReq Local request flag. + * @param req Request. * @return Prepare future. */ - public IgniteInternalFuture prepareNearTx( - final UUID nearNodeId, - final GridNearTxPrepareRequest req, - boolean locReq - ) { + public IgniteInternalFuture prepareNearTxLocal(final GridNearTxPrepareRequest req) { // Make sure not to provide Near entries to DHT cache. - if (locReq) - req.cloneEntries(); - - ClusterNode nearNode = ctx.node(nearNodeId); + req.cloneEntries(); - if (nearNode == null) { - if (txPrepareMsgLog.isDebugEnabled()) { - txPrepareMsgLog.debug("Received near prepare from node that left grid (will ignore) [" + - "txId=" + req.version() + - ", node=" + nearNodeId + ']'); - } - - return null; - } + return prepareNearTx(ctx.localNode(), req); + } + /** + * @param nearNode Node that initiated transaction. + * @param req Near prepare request. + * @return Prepare future. + */ + private IgniteInternalFuture prepareNearTx( + final ClusterNode nearNode, + final GridNearTxPrepareRequest req + ) { IgniteTxEntry firstEntry; try { @@ -350,7 +355,7 @@ public IgniteInternalFuture prepareNearTx( if (txPrepareMsgLog.isDebugEnabled()) { txPrepareMsgLog.debug("Topology version mismatch for near prepare, need remap transaction [" + "txId=" + req.version() + - ", node=" + nearNodeId + + ", node=" + nearNode.id() + ", reqTopVer=" + req.topologyVersion() + ", locTopVer=" + top.topologyVersion() + ", req=" + req + ']'); @@ -370,24 +375,24 @@ public IgniteInternalFuture prepareNearTx( req.deployInfo() != null); try { - ctx.io().send(nearNodeId, res, req.policy()); + ctx.io().send(nearNode, res, req.policy()); if (txPrepareMsgLog.isDebugEnabled()) { txPrepareMsgLog.debug("Sent remap response for near prepare [txId=" + req.version() + - ", node=" + nearNodeId + ']'); + ", node=" + nearNode.id() + ']'); } } catch (ClusterTopologyCheckedException ignored) { if (txPrepareMsgLog.isDebugEnabled()) { txPrepareMsgLog.debug("Failed to send remap response for near prepare, node failed [" + "txId=" + req.version() + - ", node=" + nearNodeId + ']'); + ", node=" + nearNode.id() + ']'); } } catch (IgniteCheckedException e) { U.error(txPrepareMsgLog, "Failed to send remap response for near prepare " + "[txId=" + req.version() + - ", node=" + nearNodeId + + ", node=" + nearNode.id() + ", req=" + req + ']', e); } From 7a6af69563bfdb9dd7043c94c82a60b8e260d595 Mon Sep 17 00:00:00 2001 From: nikolay_tikhonov Date: Thu, 27 Jul 2017 18:26:29 +0300 Subject: [PATCH 048/547] Added docker file for 2.1 version. Signed-off-by: nikolay_tikhonov --- modules/docker/2.1.0/Dockerfile | 46 +++++++++++++++++++++++++++++ modules/docker/2.1.0/run.sh | 51 +++++++++++++++++++++++++++++++++ modules/docker/Dockerfile | 2 +- 3 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 modules/docker/2.1.0/Dockerfile create mode 100644 modules/docker/2.1.0/run.sh diff --git a/modules/docker/2.1.0/Dockerfile b/modules/docker/2.1.0/Dockerfile new file mode 100644 index 0000000000000..6a0eecd4a043f --- /dev/null +++ b/modules/docker/2.1.0/Dockerfile @@ -0,0 +1,46 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Start from a Java image. +FROM java:8 + +# Ignite version +ENV IGNITE_VERSION 2.1.0 + +# Ignite home +ENV IGNITE_HOME /opt/ignite/apache-ignite-fabric-${IGNITE_VERSION}-bin + +# Do not rely on anything provided by base image(s), but be explicit, if they are installed already it is noop then +RUN apt-get update && apt-get install -y --no-install-recommends \ + unzip \ + curl \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /opt/ignite + +RUN curl https://dist.apache.org/repos/dist/release/ignite/${IGNITE_VERSION}/apache-ignite-fabric-${IGNITE_VERSION}-bin.zip -o ignite.zip \ + && unzip ignite.zip \ + && rm ignite.zip + +# Copy sh files and set permission +COPY ./run.sh $IGNITE_HOME/ + +RUN chmod +x $IGNITE_HOME/run.sh + +CMD $IGNITE_HOME/run.sh + +EXPOSE 11211 47100 47500 49112 \ No newline at end of file diff --git a/modules/docker/2.1.0/run.sh b/modules/docker/2.1.0/run.sh new file mode 100644 index 0000000000000..3aafc30ba0b9e --- /dev/null +++ b/modules/docker/2.1.0/run.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +if [ ! -z "$OPTION_LIBS" ]; then + IFS=, LIBS_LIST=("$OPTION_LIBS") + + for lib in ${LIBS_LIST[@]}; do + cp -r $IGNITE_HOME/libs/optional/"$lib"/* \ + $IGNITE_HOME/libs/ + done +fi + +if [ ! -z "$EXTERNAL_LIBS" ]; then + IFS=, LIBS_LIST=("$EXTERNAL_LIBS") + + for lib in ${LIBS_LIST[@]}; do + echo $lib >> temp + done + + wget -i temp -P $IGNITE_HOME/libs + + rm temp +fi + +QUIET="" + +if [ "$IGNITE_QUIET" = "false" ]; then + QUIET="-v" +fi + +if [ -z $CONFIG_URI ]; then + $IGNITE_HOME/bin/ignite.sh $QUIET +else + $IGNITE_HOME/bin/ignite.sh $QUIET $CONFIG_URI +fi + diff --git a/modules/docker/Dockerfile b/modules/docker/Dockerfile index 229298184f155..6a0eecd4a043f 100644 --- a/modules/docker/Dockerfile +++ b/modules/docker/Dockerfile @@ -19,7 +19,7 @@ FROM java:8 # Ignite version -ENV IGNITE_VERSION 2.0.0 +ENV IGNITE_VERSION 2.1.0 # Ignite home ENV IGNITE_HOME /opt/ignite/apache-ignite-fabric-${IGNITE_VERSION}-bin From 2941392213d2d3d9632a30d1726502d31a12e938 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Thu, 27 Jul 2017 19:00:08 +0300 Subject: [PATCH 049/547] IGNITE-2190 ScanQuery without a filter triggers object's deserialization on the server side --- .../eventstorage/GridEventStorageManager.java | 14 +++++++++ .../cache/query/GridCacheQueryManager.java | 24 +++++++++++---- .../IgniteCacheBinaryObjectsScanSelfTest.java | 9 +++++- ...heBinaryObjectsScanWithEventsSelfTest.java | 30 +++++++++++++++++++ .../IgniteBinaryCacheQueryTestSuite.java | 2 ++ 5 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheBinaryObjectsScanWithEventsSelfTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java index 944420fdf1ed9..2b9a5ee3f1b3f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java @@ -533,6 +533,20 @@ public boolean isRecordable(int type) { return type < len ? recordableEvts[type] : isUserRecordable0(type); } + /** + * Checks whether this event type has any listener. + * + * @param type Event type to check. + * @return Whether or not this event type has any listener. + */ + public boolean hasListener(int type) { + assert type > 0 : "Invalid event type: " + type; + + Listeners listeners = lsnrs.get(type); + + return (listeners != null) && (!F.isEmpty(listeners.highPriorityLsnrs) || !F.isEmpty(listeners.lsnrs)); + } + /** * Checks whether all provided events are user-recordable. *

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java index 0f47b7f08a32c..f107038803034 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java @@ -103,6 +103,7 @@ import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.A; import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.LT; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiPredicate; @@ -118,6 +119,7 @@ import org.jetbrains.annotations.Nullable; import org.jsr166.ConcurrentHashMap8; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_QUIET; import static org.apache.ignite.cache.CacheMode.LOCAL; import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_EXECUTED; import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_OBJECT_READ; @@ -1015,7 +1017,7 @@ protected void runFieldsQuery(GridCacheQueryInfo qryInfo) { metrics.addGetTimeNanos(System.nanoTime() - start); } - if (readEvt) { + if (readEvt && cctx.gridEvents().hasListener(EVT_CACHE_QUERY_OBJECT_READ)) { cctx.gridEvents().record(new CacheQueryReadEvent( cctx.localNode(), "SQL fields query result set row read.", @@ -1135,6 +1137,8 @@ protected void runQuery(GridCacheQueryInfo qryInfo) { boolean rmvIter = true; + GridCacheQueryAdapter qry = qryInfo.query(); + try { // Preparing query closures. IgniteClosure, Object> trans = @@ -1145,8 +1149,6 @@ protected void runQuery(GridCacheQueryInfo qryInfo) { injectResources(trans); injectResources(rdc); - GridCacheQueryAdapter qry = qryInfo.query(); - int pageSize = qry.pageSize(); boolean incBackups = qry.includeBackups(); @@ -1245,7 +1247,7 @@ protected void runQuery(GridCacheQueryInfo qryInfo) { K key0 = null; V val0 = null; - if (readEvt) { + if (readEvt && cctx.gridEvents().hasListener(EVT_CACHE_QUERY_OBJECT_READ)) { key0 = (K)cctx.unwrapBinaryIfNeeded(key, qry.keepBinary()); val0 = (V)cctx.unwrapBinaryIfNeeded(val, qry.keepBinary()); @@ -1372,6 +1374,18 @@ protected void runQuery(GridCacheQueryInfo qryInfo) { } } catch (Throwable e) { + if (X.hasCause(e, ClassNotFoundException.class) && !qry.keepBinary() && cctx.binaryMarshaller() && + !cctx.localNode().isClient() && !log.isQuiet()) { + LT.warn(log, "Suggestion for the cause of ClassNotFoundException"); + LT.warn(log, "To disable, set -D" + IGNITE_QUIET + "=true"); + LT.warn(log, " ^-- Ignite configured to use BinaryMarshaller but keepBinary is false for " + + "request"); + LT.warn(log, " ^-- Server node need to load definition of data classes. " + + "It can be reason of ClassNotFoundException(consider IgniteCache.withKeepBinary to fix)"); + LT.warn(log, "Refer this page for detailed information: " + + "https://apacheignite.readme.io/docs/binary-marshaller"); + } + if (!X.hasCause(e, GridDhtUnreservedPartitionException.class)) U.error(log, "Failed to run query [qry=" + qryInfo + ", node=" + cctx.nodeId() + "]", e); @@ -1471,7 +1485,7 @@ protected GridCloseableIterator scanQueryLocal(final GridCacheQueryAdapter qry, metrics.addGetTimeNanos(System.nanoTime() - start); } - if (readEvt) { + if (readEvt && cctx.gridEvents().hasListener(EVT_CACHE_QUERY_OBJECT_READ)) { cctx.gridEvents().record(new CacheQueryReadEvent<>( cctx.localNode(), "Scan query entry read.", diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheBinaryObjectsScanSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheBinaryObjectsScanSelfTest.java index e0da1f68b9873..f18aebe027556 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheBinaryObjectsScanSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheBinaryObjectsScanSelfTest.java @@ -75,7 +75,7 @@ public class IgniteCacheBinaryObjectsScanSelfTest extends GridCommonAbstractTest discoSpi.setIpFinder(IP_FINDER); cfg.setDiscoverySpi(discoSpi); - cfg.setIncludeEventTypes(new int[0]); + cfg.setIncludeEventTypes(getIncludeEventTypes()); cfg.setMarshaller(null); cfg.setPeerClassLoadingEnabled(false); @@ -89,6 +89,13 @@ public class IgniteCacheBinaryObjectsScanSelfTest extends GridCommonAbstractTest return cfg; } + /** + * @return EventTypes to record. + */ + protected int[] getIncludeEventTypes() { + return new int[0]; + } + /** * @param ldr Class loader. * @throws Exception If failed. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheBinaryObjectsScanWithEventsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheBinaryObjectsScanWithEventsSelfTest.java new file mode 100644 index 0000000000000..5746a3420a8a3 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheBinaryObjectsScanWithEventsSelfTest.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import org.apache.ignite.events.EventType; + +/** + * + */ +public class IgniteCacheBinaryObjectsScanWithEventsSelfTest extends IgniteCacheBinaryObjectsScanSelfTest { + /** {@inheritDoc} */ + @Override protected int[] getIncludeEventTypes() { + return EventType.EVTS_ALL; + } +} diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java index 1cfb345a507f6..8164fe05518f0 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java @@ -21,6 +21,7 @@ import org.apache.ignite.internal.processors.cache.BinarySerializationQuerySelfTest; import org.apache.ignite.internal.processors.cache.BinarySerializationQueryWithReflectiveSerializerSelfTest; import org.apache.ignite.internal.processors.cache.IgniteCacheBinaryObjectsScanSelfTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheBinaryObjectsScanWithEventsSelfTest; /** * Cache query suite with binary marshaller. @@ -37,6 +38,7 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(BinarySerializationQuerySelfTest.class); suite.addTestSuite(BinarySerializationQueryWithReflectiveSerializerSelfTest.class); suite.addTestSuite(IgniteCacheBinaryObjectsScanSelfTest.class); + suite.addTestSuite(IgniteCacheBinaryObjectsScanWithEventsSelfTest.class); //Should be adjusted. Not ready to be used with BinaryMarshaller. //suite.addTestSuite(GridCacheBinarySwapScanQuerySelfTest.class); From 47fea40b07857a08727ac2e062cb9a3e0f464fdc Mon Sep 17 00:00:00 2001 From: Igor Sapego Date: Thu, 27 Jul 2017 19:39:51 +0300 Subject: [PATCH 050/547] IGNITE-5771: Added Ignite::SetActive() for C++ --- .../cpp/core-test/src/cluster_test.cpp | 13 ++++++ .../cpp/core/include/ignite/ignite.h | 15 ++++++ .../ignite/impl/cluster/cluster_group_impl.h | 15 ++++++ .../core/include/ignite/impl/ignite_impl.h | 46 +++++++++---------- modules/platforms/cpp/core/src/ignite.cpp | 10 ++++ .../src/impl/cluster/cluster_group_impl.cpp | 26 ++++++++++- .../cpp/core/src/impl/ignite_impl.cpp | 24 ++++++++++ 7 files changed, 124 insertions(+), 25 deletions(-) diff --git a/modules/platforms/cpp/core-test/src/cluster_test.cpp b/modules/platforms/cpp/core-test/src/cluster_test.cpp index e9d6728bce4c9..4ee3f39e97625 100644 --- a/modules/platforms/cpp/core-test/src/cluster_test.cpp +++ b/modules/platforms/cpp/core-test/src/cluster_test.cpp @@ -83,4 +83,17 @@ BOOST_AUTO_TEST_CASE(IgniteImplForServers) BOOST_REQUIRE(clusterGroup.Get()->ForServers().IsValid()); } +BOOST_AUTO_TEST_CASE(IgniteSetActive) +{ + BOOST_REQUIRE(node.IsActive()); + + node.SetActive(false); + + BOOST_REQUIRE(!node.IsActive()); + + node.SetActive(true); + + BOOST_REQUIRE(node.IsActive()); +} + BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/modules/platforms/cpp/core/include/ignite/ignite.h b/modules/platforms/cpp/core/include/ignite/ignite.h index 07134a1a55577..e953b8fed2004 100644 --- a/modules/platforms/cpp/core/include/ignite/ignite.h +++ b/modules/platforms/cpp/core/include/ignite/ignite.h @@ -181,6 +181,21 @@ namespace ignite return cache::Cache(cacheImpl); } + /** + * Check if the Ignite grid is active. + * + * @return True if grid is active and false otherwise. + */ + bool IsActive(); + + /** + * Change Ignite grid state to active or inactive. + * + * @param active If true start activation process. If false start + * deactivation process. + */ + void SetActive(bool active); + /** * Get transactions. * diff --git a/modules/platforms/cpp/core/include/ignite/impl/cluster/cluster_group_impl.h b/modules/platforms/cpp/core/include/ignite/impl/cluster/cluster_group_impl.h index 3cfd700297f8e..d81e89919c99c 100644 --- a/modules/platforms/cpp/core/include/ignite/impl/cluster/cluster_group_impl.h +++ b/modules/platforms/cpp/core/include/ignite/impl/cluster/cluster_group_impl.h @@ -71,6 +71,21 @@ namespace ignite */ SP_ComputeImpl GetCompute(); + /** + * Check if the Ignite grid is active. + * + * @return True if grid is active and false otherwise. + */ + bool IsActive(); + + /** + * Change Ignite grid state to active or inactive. + * + * @param active If true start activation process. If false start + * deactivation process. + */ + void SetActive(bool active); + private: IGNITE_NO_COPY_ASSIGNMENT(ClusterGroupImpl); diff --git a/modules/platforms/cpp/core/include/ignite/impl/ignite_impl.h b/modules/platforms/cpp/core/include/ignite/impl/ignite_impl.h index d1763c4316a58..5461d1cd59681 100644 --- a/modules/platforms/cpp/core/include/ignite/impl/ignite_impl.h +++ b/modules/platforms/cpp/core/include/ignite/impl/ignite_impl.h @@ -182,6 +182,27 @@ namespace ignite */ SP_ComputeImpl GetCompute(); + /** + * Check if the Ignite grid is active. + * + * @return True if grid is active and false otherwise. + */ + bool IsActive() + { + return prjImpl.Get()->IsActive(); + } + + /** + * Change Ignite grid state to active or inactive. + * + * @param active If true start activation process. If false start + * deactivation process. + */ + void SetActive(bool active) + { + prjImpl.Get()->SetActive(active); + } + private: /** * Get transactions internal call. @@ -215,30 +236,7 @@ namespace ignite * @param err Error. * @param op Operation code. */ - cache::CacheImpl* GetOrCreateCache(const char* name, IgniteError& err, int32_t op) - { - SharedPointer mem = env.Get()->AllocateMemory(); - InteropMemory* mem0 = mem.Get(); - InteropOutputStream out(mem0); - BinaryWriterImpl writer(&out, env.Get()->GetTypeManager()); - BinaryRawWriter rawWriter(&writer); - - rawWriter.WriteString(name); - - out.Synchronize(); - - jobject cacheJavaRef = InStreamOutObject(op, *mem0, err); - - if (!cacheJavaRef) - { - return NULL; - } - - char* name0 = common::CopyChars(name); - - return new cache::CacheImpl(name0, env, cacheJavaRef); - } - + cache::CacheImpl* GetOrCreateCache(const char* name, IgniteError& err, int32_t op); }; } } diff --git a/modules/platforms/cpp/core/src/ignite.cpp b/modules/platforms/cpp/core/src/ignite.cpp index 9c42f1d117d1c..6eaae01a339a5 100644 --- a/modules/platforms/cpp/core/src/ignite.cpp +++ b/modules/platforms/cpp/core/src/ignite.cpp @@ -45,6 +45,16 @@ namespace ignite return impl.Get()->GetConfiguration(); } + bool Ignite::IsActive() + { + return impl.Get()->IsActive(); + } + + void Ignite::SetActive(bool active) + { + impl.Get()->SetActive(active); + } + transactions::Transactions Ignite::GetTransactions() { using ignite::common::concurrent::SharedPointer; diff --git a/modules/platforms/cpp/core/src/impl/cluster/cluster_group_impl.cpp b/modules/platforms/cpp/core/src/impl/cluster/cluster_group_impl.cpp index c34e828b4a76f..91f9d304f2663 100644 --- a/modules/platforms/cpp/core/src/impl/cluster/cluster_group_impl.cpp +++ b/modules/platforms/cpp/core/src/impl/cluster/cluster_group_impl.cpp @@ -30,7 +30,11 @@ namespace ignite { enum Type { - FOR_SERVERS = 23 + FOR_SERVERS = 23, + + SET_ACTIVE = 28, + + IS_ACTIVE = 29 }; }; @@ -61,6 +65,26 @@ namespace ignite return computeImpl; } + bool ClusterGroupImpl::IsActive() + { + IgniteError err; + + int64_t res = OutInOpLong(Command::IS_ACTIVE, 0, err); + + IgniteError::ThrowIfNeeded(err); + + return res == 1; + } + + void ClusterGroupImpl::SetActive(bool active) + { + IgniteError err; + + int64_t res = OutInOpLong(Command::SET_ACTIVE, active ? 1 : 0, err); + + IgniteError::ThrowIfNeeded(err); + } + SP_ClusterGroupImpl ClusterGroupImpl::FromTarget(jobject javaRef) { return SP_ClusterGroupImpl(new ClusterGroupImpl(GetEnvironmentPointer(), javaRef)); diff --git a/modules/platforms/cpp/core/src/impl/ignite_impl.cpp b/modules/platforms/cpp/core/src/impl/ignite_impl.cpp index f7ff1859e05a5..f2132d482f5cd 100644 --- a/modules/platforms/cpp/core/src/impl/ignite_impl.cpp +++ b/modules/platforms/cpp/core/src/impl/ignite_impl.cpp @@ -91,5 +91,29 @@ namespace ignite return res; } + + cache::CacheImpl* IgniteImpl::GetOrCreateCache(const char* name, IgniteError& err, int32_t op) + { + SharedPointer mem = env.Get()->AllocateMemory(); + InteropMemory* mem0 = mem.Get(); + InteropOutputStream out(mem0); + BinaryWriterImpl writer(&out, env.Get()->GetTypeManager()); + BinaryRawWriter rawWriter(&writer); + + rawWriter.WriteString(name); + + out.Synchronize(); + + jobject cacheJavaRef = InStreamOutObject(op, *mem0, err); + + if (!cacheJavaRef) + { + return NULL; + } + + char* name0 = common::CopyChars(name); + + return new cache::CacheImpl(name0, env, cacheJavaRef); + } } } From 98a524067b2d2349c558256ee38efdf81e0ba7ff Mon Sep 17 00:00:00 2001 From: Igor Sapego Date: Thu, 27 Jul 2017 19:39:51 +0300 Subject: [PATCH 051/547] IGNITE-5771: Added Ignite::SetActive() for C++ (cherry picked from commit 47fea40) --- .../cpp/core-test/src/cluster_test.cpp | 13 ++++++++++ .../cpp/core/include/ignite/ignite.h | 15 +++++++++++ .../ignite/impl/cluster/cluster_group_impl.h | 15 +++++++++++ .../core/include/ignite/impl/ignite_impl.h | 23 +++++++++++++++- modules/platforms/cpp/core/src/ignite.cpp | 10 +++++++ .../src/impl/cluster/cluster_group_impl.cpp | 26 ++++++++++++++++++- 6 files changed, 100 insertions(+), 2 deletions(-) diff --git a/modules/platforms/cpp/core-test/src/cluster_test.cpp b/modules/platforms/cpp/core-test/src/cluster_test.cpp index e9d6728bce4c9..4ee3f39e97625 100644 --- a/modules/platforms/cpp/core-test/src/cluster_test.cpp +++ b/modules/platforms/cpp/core-test/src/cluster_test.cpp @@ -83,4 +83,17 @@ BOOST_AUTO_TEST_CASE(IgniteImplForServers) BOOST_REQUIRE(clusterGroup.Get()->ForServers().IsValid()); } +BOOST_AUTO_TEST_CASE(IgniteSetActive) +{ + BOOST_REQUIRE(node.IsActive()); + + node.SetActive(false); + + BOOST_REQUIRE(!node.IsActive()); + + node.SetActive(true); + + BOOST_REQUIRE(node.IsActive()); +} + BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/modules/platforms/cpp/core/include/ignite/ignite.h b/modules/platforms/cpp/core/include/ignite/ignite.h index b3b06f034d57f..135f21516005c 100644 --- a/modules/platforms/cpp/core/include/ignite/ignite.h +++ b/modules/platforms/cpp/core/include/ignite/ignite.h @@ -181,6 +181,21 @@ namespace ignite return cache::Cache(cacheImpl); } + /** + * Check if the Ignite grid is active. + * + * @return True if grid is active and false otherwise. + */ + bool IsActive(); + + /** + * Change Ignite grid state to active or inactive. + * + * @param active If true start activation process. If false start + * deactivation process. + */ + void SetActive(bool active); + /** * Get transactions. * diff --git a/modules/platforms/cpp/core/include/ignite/impl/cluster/cluster_group_impl.h b/modules/platforms/cpp/core/include/ignite/impl/cluster/cluster_group_impl.h index 3cfd700297f8e..d81e89919c99c 100644 --- a/modules/platforms/cpp/core/include/ignite/impl/cluster/cluster_group_impl.h +++ b/modules/platforms/cpp/core/include/ignite/impl/cluster/cluster_group_impl.h @@ -71,6 +71,21 @@ namespace ignite */ SP_ComputeImpl GetCompute(); + /** + * Check if the Ignite grid is active. + * + * @return True if grid is active and false otherwise. + */ + bool IsActive(); + + /** + * Change Ignite grid state to active or inactive. + * + * @param active If true start activation process. If false start + * deactivation process. + */ + void SetActive(bool active); + private: IGNITE_NO_COPY_ASSIGNMENT(ClusterGroupImpl); diff --git a/modules/platforms/cpp/core/include/ignite/impl/ignite_impl.h b/modules/platforms/cpp/core/include/ignite/impl/ignite_impl.h index baddec49b1038..ea192e2cf846c 100644 --- a/modules/platforms/cpp/core/include/ignite/impl/ignite_impl.h +++ b/modules/platforms/cpp/core/include/ignite/impl/ignite_impl.h @@ -82,7 +82,7 @@ namespace ignite * @param name Cache name. * @param err Error. */ - template + template cache::CacheImpl* GetCache(const char* name, IgniteError& err) { ignite::jni::java::JniErrorInfo jniErr; @@ -209,6 +209,27 @@ namespace ignite */ SP_ComputeImpl GetCompute(); + /** + * Check if the Ignite grid is active. + * + * @return True if grid is active and false otherwise. + */ + bool IsActive() + { + return prjImpl.Get()->IsActive(); + } + + /** + * Change Ignite grid state to active or inactive. + * + * @param active If true start activation process. If false start + * deactivation process. + */ + void SetActive(bool active) + { + prjImpl.Get()->SetActive(active); + } + private: /** * Get transactions internal call. diff --git a/modules/platforms/cpp/core/src/ignite.cpp b/modules/platforms/cpp/core/src/ignite.cpp index 9c42f1d117d1c..6eaae01a339a5 100644 --- a/modules/platforms/cpp/core/src/ignite.cpp +++ b/modules/platforms/cpp/core/src/ignite.cpp @@ -45,6 +45,16 @@ namespace ignite return impl.Get()->GetConfiguration(); } + bool Ignite::IsActive() + { + return impl.Get()->IsActive(); + } + + void Ignite::SetActive(bool active) + { + impl.Get()->SetActive(active); + } + transactions::Transactions Ignite::GetTransactions() { using ignite::common::concurrent::SharedPointer; diff --git a/modules/platforms/cpp/core/src/impl/cluster/cluster_group_impl.cpp b/modules/platforms/cpp/core/src/impl/cluster/cluster_group_impl.cpp index c34e828b4a76f..91f9d304f2663 100644 --- a/modules/platforms/cpp/core/src/impl/cluster/cluster_group_impl.cpp +++ b/modules/platforms/cpp/core/src/impl/cluster/cluster_group_impl.cpp @@ -30,7 +30,11 @@ namespace ignite { enum Type { - FOR_SERVERS = 23 + FOR_SERVERS = 23, + + SET_ACTIVE = 28, + + IS_ACTIVE = 29 }; }; @@ -61,6 +65,26 @@ namespace ignite return computeImpl; } + bool ClusterGroupImpl::IsActive() + { + IgniteError err; + + int64_t res = OutInOpLong(Command::IS_ACTIVE, 0, err); + + IgniteError::ThrowIfNeeded(err); + + return res == 1; + } + + void ClusterGroupImpl::SetActive(bool active) + { + IgniteError err; + + int64_t res = OutInOpLong(Command::SET_ACTIVE, active ? 1 : 0, err); + + IgniteError::ThrowIfNeeded(err); + } + SP_ClusterGroupImpl ClusterGroupImpl::FromTarget(jobject javaRef) { return SP_ClusterGroupImpl(new ClusterGroupImpl(GetEnvironmentPointer(), javaRef)); From 6031ed85aa0242e8a56dee43c14c73bcafa7fab7 Mon Sep 17 00:00:00 2001 From: vsisko Date: Fri, 28 Jul 2017 09:54:07 +0700 Subject: [PATCH 052/547] GG-12544 Removed limitation for stacktrace showing. --- .../ignite/internal/visor/debug/VisorThreadInfo.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/debug/VisorThreadInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/debug/VisorThreadInfo.java index f48e6c141fc0e..9b0dd892cf1c2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/debug/VisorThreadInfo.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/debug/VisorThreadInfo.java @@ -35,9 +35,6 @@ public class VisorThreadInfo extends VisorDataTransferObject { /** */ private static final long serialVersionUID = 0L; - /** */ - private static final int MAX_FRAMES = 8; - /** Thread name. */ private String name; @@ -308,9 +305,7 @@ public List getLockedMonitors() { sb.append('\n'); - int maxFrames = Math.min(stackTrace.size(), MAX_FRAMES); - - for (int i = 0; i < maxFrames; i++) { + for (int i = 0; i < stackTrace.size(); i++) { StackTraceElement ste = stackTrace.get(i); sb.append("\tat ").append(ste).append('\n'); @@ -339,9 +334,6 @@ public List getLockedMonitors() { } } - if (maxFrames < stackTrace.size()) - sb.append("\t...").append('\n'); - if (!F.isEmpty(locks)) { sb.append("\n\tNumber of locked synchronizers = ").append(locks.size()).append('\n'); From 89bba2fa2c423d5713c8412ba0069b869005694c Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Fri, 28 Jul 2017 10:06:16 +0300 Subject: [PATCH 053/547] IGNITE-5769 Abstract away .NET->Java calls This closes #2352 --- .../platform/PlatformTargetProxy.java | 11 + .../platform/PlatformTargetProxyImpl.java | 79 +- .../plugin/PlatformTestPluginTarget.java | 7 +- .../cpp/jni/include/ignite/jni/exports.h | 1 + .../cpp/jni/include/ignite/jni/java.h | 5 +- .../platforms/cpp/jni/project/vs/module.def | 1 + modules/platforms/cpp/jni/src/exports.cpp | 4 + modules/platforms/cpp/jni/src/java.cpp | 19 +- .../Plugin/PluginTest.cs | 13 +- .../Apache.Ignite.Core.Tests/TestUtils.cs | 7 +- .../Apache.Ignite.Core.csproj | 5 +- .../dotnet/Apache.Ignite.Core/Ignition.cs | 9 +- .../Impl/Binary/BinaryProcessor.cs | 6 +- .../Impl/Binary/BinaryWriterExtensions.cs | 107 ++ .../Affinity/PlatformAffinityFunction.cs | 7 +- .../Impl/Cache/CacheAffinityImpl.cs | 18 +- .../Impl/Cache/CacheEnumerator.cs | 8 +- .../Impl/Cache/CacheImpl.cs | 113 +- .../Impl/Cache/Query/AbstractQueryCursor.cs | 12 +- .../Continuous/ContinuousQueryHandleImpl.cs | 12 +- .../Impl/Cache/Query/FieldsQueryCursor.cs | 6 +- .../Impl/Cache/Query/QueryCursor.cs | 5 +- .../Impl/Cluster/ClusterGroupImpl.cs | 76 +- .../Impl/Common/DelegateTypeDescriptor.cs | 9 +- .../Impl/Common/Listenable.cs | 8 +- .../Impl/Compute/ComputeImpl.cs | 12 +- .../Impl/DataStructures/AtomicLong.cs | 9 +- .../Impl/DataStructures/AtomicReference.cs | 8 +- .../Impl/DataStructures/AtomicSequence.cs | 9 +- .../Impl/Datastream/DataStreamerImpl.cs | 8 +- .../Impl/Datastream/StreamReceiverHolder.cs | 13 +- .../Apache.Ignite.Core/Impl/Events/Events.cs | 11 +- .../Impl/IPlatformTargetInternal.cs | 102 ++ .../dotnet/Apache.Ignite.Core/Impl/Ignite.cs | 48 +- .../Impl/Messaging/Messaging.cs | 10 +- .../Impl/PlatformDisposableTargetAdapter.cs | 75 ++ .../Impl/PlatformJniTarget.cs | 536 ++++++++ .../Apache.Ignite.Core/Impl/PlatformTarget.cs | 1086 ----------------- .../Impl/PlatformTargetAdapter.cs | 534 ++++++++ .../Impl/Services/Services.cs | 19 +- .../Impl/Transactions/TransactionsImpl.cs | 29 +- .../Impl/Unmanaged/IgniteJniNativeMethods.cs | 3 + .../Impl/Unmanaged/UnmanagedCallbacks.cs | 10 +- .../Impl/Unmanaged/UnmanagedUtils.cs | 7 + .../Interop/IPlatformTarget.cs | 15 + 45 files changed, 1690 insertions(+), 1402 deletions(-) create mode 100644 modules/platforms/dotnet/Apache.Ignite.Core/Impl/IPlatformTargetInternal.cs create mode 100644 modules/platforms/dotnet/Apache.Ignite.Core/Impl/PlatformDisposableTargetAdapter.cs create mode 100644 modules/platforms/dotnet/Apache.Ignite.Core/Impl/PlatformJniTarget.cs delete mode 100644 modules/platforms/dotnet/Apache.Ignite.Core/Impl/PlatformTarget.cs create mode 100644 modules/platforms/dotnet/Apache.Ignite.Core/Impl/PlatformTargetAdapter.cs diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformTargetProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformTargetProxy.java index 1ee57cb98141c..29de311d3990b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformTargetProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformTargetProxy.java @@ -104,6 +104,17 @@ Object inObjectStreamOutObjectStream(int type, @Nullable Object arg, long inMemP */ void inStreamAsync(int type, long memPtr) throws Exception; + /** + * Asynchronous operation accepting memory stream and returning PlatformListenableTarget. + * Supports cancellable async operations. + * + * @param type Operation type. + * @param memPtr Memory pointer. + * @return Result. + * @throws Exception If case of failure. + */ + Object inStreamOutObjectAsync(int type, long memPtr) throws Exception; + /** * Returns the underlying target. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformTargetProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformTargetProxyImpl.java index 44044b1b0de98..b472275f5e28e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformTargetProxyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformTargetProxyImpl.java @@ -23,6 +23,8 @@ import org.apache.ignite.internal.processors.platform.memory.PlatformMemory; import org.apache.ignite.internal.processors.platform.memory.PlatformOutputStream; import org.apache.ignite.internal.processors.platform.utils.PlatformFutureUtils; +import org.apache.ignite.internal.processors.platform.utils.PlatformListenable; +import org.apache.ignite.internal.processors.platform.utils.PlatformListenableTarget; import org.apache.ignite.lang.IgniteFuture; /** @@ -109,37 +111,16 @@ public PlatformTargetProxyImpl(PlatformTarget target, PlatformContext platformCt /** {@inheritDoc} */ @Override public void inStreamAsync(int type, long memPtr) throws Exception { - try (PlatformMemory mem = platformCtx.memory().get(memPtr)) { - BinaryRawReaderEx reader = platformCtx.reader(mem); - - long futId = reader.readLong(); - int futTyp = reader.readInt(); - - final PlatformAsyncResult res = target.processInStreamAsync(type, reader); - - if (res == null) - throw new IgniteException("PlatformTarget.processInStreamAsync should not return null."); - - IgniteFuture fut = res.future(); + inStreamOutListenableAsync(type, memPtr); + } - if (fut == null) - throw new IgniteException("PlatformAsyncResult.future() should not return null."); + /** {@inheritDoc} */ + @Override public Object inStreamOutObjectAsync(int type, long memPtr) throws Exception { + PlatformListenable listenable = inStreamOutListenableAsync(type, memPtr); - PlatformFutureUtils.listen(platformCtx, fut, futId, futTyp, new PlatformFutureUtils.Writer() { - /** {@inheritDoc} */ - @Override public void write(BinaryRawWriterEx writer, Object obj, Throwable err) { - res.write(writer, obj); - } + PlatformListenableTarget target = new PlatformListenableTarget(listenable, platformCtx); - /** {@inheritDoc} */ - @Override public boolean canWrite(Object obj, Throwable err) { - return err == null; - } - }, target); - } - catch (Exception e) { - throw target.convertException(e); - } + return wrapProxy(target); } /** {@inheritDoc} */ @@ -234,4 +215,46 @@ private Object wrapProxy(PlatformTarget obj) { private PlatformTarget unwrapProxy(Object obj) { return obj == null ? null : ((PlatformTargetProxyImpl)obj).target; } + + /** + * Performs asyncronous operation. + * + * @param type Type. + * @param memPtr Stream pointer. + * @return Listenable. + * @throws Exception On error. + */ + private PlatformListenable inStreamOutListenableAsync(int type, long memPtr) throws Exception { + try (PlatformMemory mem = platformCtx.memory().get(memPtr)) { + BinaryRawReaderEx reader = platformCtx.reader(mem); + + long futId = reader.readLong(); + int futTyp = reader.readInt(); + + final PlatformAsyncResult res = target.processInStreamAsync(type, reader); + + if (res == null) + throw new IgniteException("PlatformTarget.processInStreamAsync should not return null."); + + IgniteFuture fut = res.future(); + + if (fut == null) + throw new IgniteException("PlatformAsyncResult.future() should not return null."); + + return PlatformFutureUtils.listen(platformCtx, fut, futId, futTyp, new PlatformFutureUtils.Writer() { + /** {@inheritDoc} */ + @Override public void write(BinaryRawWriterEx writer, Object obj, Throwable err) { + res.write(writer, obj); + } + + /** {@inheritDoc} */ + @Override public boolean canWrite(Object obj, Throwable err) { + return err == null; + } + }, target); + } + catch (Exception e) { + throw target.convertException(e); + } + } } diff --git a/modules/core/src/test/java/org/apache/ignite/platform/plugin/PlatformTestPluginTarget.java b/modules/core/src/test/java/org/apache/ignite/platform/plugin/PlatformTestPluginTarget.java index 7e69425c5b20c..8c1cbe94996bf 100644 --- a/modules/core/src/test/java/org/apache/ignite/platform/plugin/PlatformTestPluginTarget.java +++ b/modules/core/src/test/java/org/apache/ignite/platform/plugin/PlatformTestPluginTarget.java @@ -146,7 +146,12 @@ private String invokeCallback(String val) { case 1: { // Async upper case. final String val = reader.readString(); - final GridFutureAdapter fa = new GridFutureAdapter<>(); + + final GridFutureAdapter fa = new GridFutureAdapter() { + @Override public boolean cancel() throws IgniteCheckedException { + return onCancelled(); + } + }; new Thread(new Runnable() { @Override public void run() { diff --git a/modules/platforms/cpp/jni/include/ignite/jni/exports.h b/modules/platforms/cpp/jni/include/ignite/jni/exports.h index ea0c32a072ad7..0580d19a9b259 100644 --- a/modules/platforms/cpp/jni/include/ignite/jni/exports.h +++ b/modules/platforms/cpp/jni/include/ignite/jni/exports.h @@ -38,6 +38,7 @@ extern "C" { void IGNITE_CALL IgniteTargetOutStream(gcj::JniContext* ctx, void* obj, int opType, long long memPtr); void* IGNITE_CALL IgniteTargetOutObject(gcj::JniContext* ctx, void* obj, int opType); void IGNITE_CALL IgniteTargetInStreamAsync(gcj::JniContext* ctx, void* obj, int opType, long long memPtr); + void* IGNITE_CALL IgniteTargetInStreamOutObjectAsync(gcj::JniContext* ctx, void* obj, int opType, long long memPtr); void* IGNITE_CALL IgniteAcquire(gcj::JniContext* ctx, void* obj); void IGNITE_CALL IgniteRelease(void* obj); diff --git a/modules/platforms/cpp/jni/include/ignite/jni/java.h b/modules/platforms/cpp/jni/include/ignite/jni/java.h index c170a5bb56755..c713e811ad733 100644 --- a/modules/platforms/cpp/jni/include/ignite/jni/java.h +++ b/modules/platforms/cpp/jni/include/ignite/jni/java.h @@ -173,9 +173,6 @@ namespace ignite jmethodID m_PlatformIgnition_stop; jmethodID m_PlatformIgnition_stopAll; - jclass c_PlatformProcessor; - jmethodID m_PlatformProcessor_releaseStart; - jclass c_PlatformTarget; jmethodID m_PlatformTarget_inLongOutLong; jmethodID m_PlatformTarget_inStreamOutLong; @@ -183,6 +180,7 @@ namespace ignite jmethodID m_PlatformTarget_outStream; jmethodID m_PlatformTarget_outObject; jmethodID m_PlatformTarget_inStreamAsync; + jmethodID m_PlatformTarget_inStreamOutObjectAsync; jmethodID m_PlatformTarget_inStreamOutStream; jmethodID m_PlatformTarget_inObjectStreamOutObjectStream; @@ -325,6 +323,7 @@ namespace ignite void TargetOutStream(jobject obj, int opType, long long memPtr, JniErrorInfo* errInfo = NULL); jobject TargetOutObject(jobject obj, int opType, JniErrorInfo* errInfo = NULL); void TargetInStreamAsync(jobject obj, int type, long long memPtr, JniErrorInfo* errInfo = NULL); + jobject TargetInStreamOutObjectAsync(jobject obj, int type, long long memPtr, JniErrorInfo* errInfo = NULL); jobject CacheOutOpQueryCursor(jobject obj, int type, long long memPtr, JniErrorInfo* errInfo = NULL); jobject CacheOutOpContinuousQuery(jobject obj, int type, long long memPtr, JniErrorInfo* errInfo = NULL); diff --git a/modules/platforms/cpp/jni/project/vs/module.def b/modules/platforms/cpp/jni/project/vs/module.def index 53e7e423f7a17..1407f821bc10c 100644 --- a/modules/platforms/cpp/jni/project/vs/module.def +++ b/modules/platforms/cpp/jni/project/vs/module.def @@ -12,6 +12,7 @@ IgniteTargetInStreamOutStream @20 IgniteTargetInObjectStreamOutObjectStream @21 IgniteTargetInLongOutLong @24 IgniteTargetInStreamAsync @25 +IgniteTargetInStreamOutObjectAsync @26 IgniteAcquire @80 IgniteRelease @81 IgniteThrowToJava @82 diff --git a/modules/platforms/cpp/jni/src/exports.cpp b/modules/platforms/cpp/jni/src/exports.cpp index 9b7defd52dd18..aeb68abb24d7a 100644 --- a/modules/platforms/cpp/jni/src/exports.cpp +++ b/modules/platforms/cpp/jni/src/exports.cpp @@ -74,6 +74,10 @@ extern "C" { ctx->TargetInStreamAsync(static_cast(obj), opType, memPtr); } + void* IGNITE_CALL IgniteTargetInStreamOutObjectAsync(gcj::JniContext* ctx, void* obj, int opType, long long memPtr) { + return ctx->TargetInStreamOutObjectAsync(static_cast(obj), opType, memPtr); + } + void* IGNITE_CALL IgniteAcquire(gcj::JniContext* ctx, void* obj) { return ctx->Acquire(static_cast(obj)); } diff --git a/modules/platforms/cpp/jni/src/java.cpp b/modules/platforms/cpp/jni/src/java.cpp index 7eadec0a27d69..ac4ba6345d5c2 100644 --- a/modules/platforms/cpp/jni/src/java.cpp +++ b/modules/platforms/cpp/jni/src/java.cpp @@ -221,9 +221,6 @@ namespace ignite const char* C_PLATFORM_NO_CALLBACK_EXCEPTION = "org/apache/ignite/internal/processors/platform/PlatformNoCallbackException"; - const char* C_PLATFORM_PROCESSOR = "org/apache/ignite/internal/processors/platform/PlatformProcessor"; - JniMethod M_PLATFORM_PROCESSOR_RELEASE_START = JniMethod("releaseStart", "()V", false); - const char* C_PLATFORM_TARGET = "org/apache/ignite/internal/processors/platform/PlatformTargetProxy"; JniMethod M_PLATFORM_TARGET_IN_LONG_OUT_LONG = JniMethod("inLongOutLong", "(IJ)J", false); JniMethod M_PLATFORM_TARGET_IN_STREAM_OUT_LONG = JniMethod("inStreamOutLong", "(IJ)J", false); @@ -233,6 +230,7 @@ namespace ignite JniMethod M_PLATFORM_TARGET_OUT_STREAM = JniMethod("outStream", "(IJ)V", false); JniMethod M_PLATFORM_TARGET_OUT_OBJECT = JniMethod("outObject", "(I)Ljava/lang/Object;", false); JniMethod M_PLATFORM_TARGET_IN_STREAM_ASYNC = JniMethod("inStreamAsync", "(IJ)V", false); + JniMethod M_PLATFORM_TARGET_IN_STREAM_OUT_OBJECT_ASYNC = JniMethod("inStreamOutObjectAsync", "(IJ)Ljava/lang/Object;", false); const char* C_PLATFORM_CALLBACK_UTILS = "org/apache/ignite/internal/processors/platform/callback/PlatformCallbackUtils"; @@ -449,9 +447,6 @@ namespace ignite m_PlatformIgnition_stop = FindMethod(env, c_PlatformIgnition, M_PLATFORM_IGNITION_STOP); m_PlatformIgnition_stopAll = FindMethod(env, c_PlatformIgnition, M_PLATFORM_IGNITION_STOP_ALL); - c_PlatformProcessor = FindClass(env, C_PLATFORM_PROCESSOR); - m_PlatformProcessor_releaseStart = FindMethod(env, c_PlatformProcessor, M_PLATFORM_PROCESSOR_RELEASE_START); - c_PlatformTarget = FindClass(env, C_PLATFORM_TARGET); m_PlatformTarget_inLongOutLong = FindMethod(env, c_PlatformTarget, M_PLATFORM_TARGET_IN_LONG_OUT_LONG); m_PlatformTarget_inStreamOutLong = FindMethod(env, c_PlatformTarget, M_PLATFORM_TARGET_IN_STREAM_OUT_LONG); @@ -461,6 +456,7 @@ namespace ignite m_PlatformTarget_inStreamOutStream = FindMethod(env, c_PlatformTarget, M_PLATFORM_TARGET_IN_STREAM_OUT_STREAM); m_PlatformTarget_inObjectStreamOutObjectStream = FindMethod(env, c_PlatformTarget, M_PLATFORM_TARGET_IN_OBJECT_STREAM_OUT_OBJECT_STREAM); m_PlatformTarget_inStreamAsync = FindMethod(env, c_PlatformTarget, M_PLATFORM_TARGET_IN_STREAM_ASYNC); + m_PlatformTarget_inStreamOutObjectAsync = FindMethod(env, c_PlatformTarget, M_PLATFORM_TARGET_IN_STREAM_OUT_OBJECT_ASYNC); c_PlatformUtils = FindClass(env, C_PLATFORM_UTILS); m_PlatformUtils_reallocate = FindMethod(env, c_PlatformUtils, M_PLATFORM_UTILS_REALLOC); @@ -473,7 +469,6 @@ namespace ignite void JniMembers::Destroy(JNIEnv* env) { DeleteClass(env, c_IgniteException); DeleteClass(env, c_PlatformIgnition); - DeleteClass(env, c_PlatformProcessor); DeleteClass(env, c_PlatformTarget); DeleteClass(env, c_PlatformUtils); } @@ -894,6 +889,16 @@ namespace ignite ExceptionCheck(env, err); } + jobject JniContext::TargetInStreamOutObjectAsync(jobject obj, int opType, long long memPtr, JniErrorInfo* err) { + JNIEnv* env = Attach(); + + jobject res = env->CallObjectMethod(obj, jvm->GetMembers().m_PlatformTarget_inStreamOutObjectAsync, opType, memPtr); + + ExceptionCheck(env, err); + + return LocalToGlobal(env, res); + } + jobject JniContext::CacheOutOpQueryCursor(jobject obj, int type, long long memPtr, JniErrorInfo* err) { JNIEnv* env = Attach(); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Plugin/PluginTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Plugin/PluginTest.cs index 00b1cca6378e2..1cb2faeb55632 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Plugin/PluginTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Plugin/PluginTest.cs @@ -21,6 +21,8 @@ namespace Apache.Ignite.Core.Tests.Plugin using System.Collections.Generic; using System.IO; using System.Linq; + using System.Threading; + using System.Threading.Tasks; using Apache.Ignite.Core.Binary; using Apache.Ignite.Core.Common; using Apache.Ignite.Core.Interop; @@ -142,13 +144,22 @@ private static void CheckResourceInjection(IPluginContext w.WriteString("foo"), r => r.ReadString(), cts.Token); + Assert.IsFalse(task.IsCompleted); + cts.Cancel(); + Assert.IsTrue(task.IsCanceled); + var aex = Assert.Throws(() => { asyncRes = task.Result; }); + Assert.IsInstanceOf(aex.GetBaseException()); + // Async operation with exception in entry point. Assert.Throws(() => target.DoOutOpAsync(2, null, null)); // Async operation with exception in future. var errTask = target.DoOutOpAsync(3, null, null); Assert.IsFalse(errTask.IsCompleted); - var aex = Assert.Throws(() => errTask.Wait()); + aex = Assert.Throws(() => errTask.Wait()); Assert.IsInstanceOf(aex.InnerExceptions.Single()); // Throws custom mapped exception. diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/TestUtils.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/TestUtils.cs index 6e0a49732c77f..4b171b0a0dc96 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/TestUtils.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/TestUtils.cs @@ -365,12 +365,11 @@ public static void RunTestInNewProcess(string fixtureName, string testName) }; var proc = System.Diagnostics.Process.Start(procStart); - Assert.IsNotNull(proc); - Console.WriteLine(proc.StandardOutput.ReadToEnd()); - Console.WriteLine(proc.StandardError.ReadToEnd()); - Assert.IsTrue(proc.WaitForExit(15000)); + IgniteProcess.AttachProcessConsoleReader(proc); + + Assert.IsTrue(proc.WaitForExit(19000)); Assert.AreEqual(0, proc.ExitCode); } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj index 76132c3423f69..c444ed0f21588 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj @@ -99,7 +99,10 @@ + + + @@ -385,7 +388,7 @@ - + diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Ignition.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Ignition.cs index 44ebef3b497ad..568eea7a0c4bb 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Ignition.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Ignition.cs @@ -239,7 +239,7 @@ public static unsafe IIgnite Start(IgniteConfiguration cfg) // 3. Create startup object which will guide us through the rest of the process. _startup = new Startup(cfg, cbs); - IUnmanagedTarget interopProc = null; + PlatformJniTarget interopProc = null; try { @@ -249,7 +249,7 @@ public static unsafe IIgnite Start(IgniteConfiguration cfg) // 5. At this point start routine is finished. We expect STARTUP object to have all necessary data. var node = _startup.Ignite; - interopProc = node.InteropProcessor; + interopProc = (PlatformJniTarget)node.InteropProcessor; var javaLogger = log as JavaLogger; if (javaLogger != null) @@ -279,7 +279,7 @@ public static unsafe IIgnite Start(IgniteConfiguration cfg) // 2. Stop Ignite node if it was started. if (interopProc != null) - UU.IgnitionStop(interopProc.Context, gridName, true); + UU.IgnitionStop(interopProc.Target.Context, gridName, true); // 3. Throw error further (use startup error if exists because it is more precise). if (_startup.Error != null) @@ -466,7 +466,8 @@ internal static void OnStart(IUnmanagedTarget interopProc, IBinaryStream stream) if (Nodes.ContainsKey(new NodeKey(name))) throw new IgniteException("Ignite with the same name already started: " + name); - _startup.Ignite = new Ignite(_startup.Configuration, _startup.Name, interopProc, _startup.Marshaller, + _startup.Ignite = new Ignite(_startup.Configuration, _startup.Name, + new PlatformJniTarget(interopProc, _startup.Marshaller), _startup.Marshaller, _startup.LifecycleHandlers, _startup.Callbacks); } catch (Exception e) diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryProcessor.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryProcessor.cs index b8937c90faeac..69056b33d7197 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryProcessor.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryProcessor.cs @@ -21,12 +21,11 @@ namespace Apache.Ignite.Core.Impl.Binary using System.Diagnostics; using Apache.Ignite.Core.Binary; using Apache.Ignite.Core.Impl.Binary.Metadata; - using Apache.Ignite.Core.Impl.Unmanaged; /// /// Binary metadata processor, delegates to PlatformBinaryProcessor in Java. /// - internal class BinaryProcessor : PlatformTarget + internal class BinaryProcessor : PlatformTargetAdapter { /// /// Op codes. @@ -46,8 +45,7 @@ private enum Op /// Initializes a new instance of the class. /// /// Target. - /// Marshaller. - public BinaryProcessor(IUnmanagedTarget target, Marshaller marsh) : base(target, marsh) + public BinaryProcessor(IPlatformTargetInternal target) : base(target) { // No-op. } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryWriterExtensions.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryWriterExtensions.cs index 64bfa35dcd41a..3dc8a962107df 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryWriterExtensions.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryWriterExtensions.cs @@ -18,6 +18,8 @@ namespace Apache.Ignite.Core.Impl.Binary { using System; + using System.Collections.Generic; + using System.IO; using Apache.Ignite.Core.Binary; /// @@ -75,5 +77,110 @@ public static void WriteTimeSpanAsLongNullable(this IBinaryRawWriter writer, Tim writer.WriteBoolean(false); } + /// + /// Write collection. + /// + /// Writer. + /// Values. + /// A transform function to apply to each element. + /// The same writer for chaining. + private static void WriteCollection(this BinaryWriter writer, ICollection vals, + Func selector) + { + writer.WriteInt(vals.Count); + + if (selector == null) + { + foreach (var val in vals) + writer.Write(val); + } + else + { + foreach (var val in vals) + writer.Write(selector(val)); + } + } + + /// + /// Write enumerable. + /// + /// Writer. + /// Values. + /// The same writer for chaining. + public static void WriteEnumerable(this BinaryWriter writer, IEnumerable vals) + { + WriteEnumerable(writer, vals, null); + } + + /// + /// Write enumerable. + /// + /// Writer. + /// Values. + /// A transform function to apply to each element. + /// The same writer for chaining. + public static void WriteEnumerable(this BinaryWriter writer, IEnumerable vals, + Func selector) + { + var col = vals as ICollection; + + if (col != null) + { + WriteCollection(writer, col, selector); + return; + } + + var stream = writer.Stream; + + var pos = stream.Position; + + stream.Seek(4, SeekOrigin.Current); + + var size = 0; + + if (selector == null) + { + foreach (var val in vals) + { + writer.Write(val); + + size++; + } + } + else + { + foreach (var val in vals) + { + writer.Write(selector(val)); + + size++; + } + } + + stream.WriteInt(pos, size); + } + + /// + /// Write dictionary. + /// + /// Writer. + /// Values. + public static void WriteDictionary(this BinaryWriter writer, IEnumerable> vals) + { + var pos = writer.Stream.Position; + writer.WriteInt(0); // Reserve count. + + int cnt = 0; + + foreach (var pair in vals) + { + writer.Write(pair.Key); + writer.Write(pair.Value); + + cnt++; + } + + writer.Stream.WriteInt(pos, cnt); + } } } \ No newline at end of file diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Affinity/PlatformAffinityFunction.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Affinity/PlatformAffinityFunction.cs index d33580425797d..08c31a6eb786f 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Affinity/PlatformAffinityFunction.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Affinity/PlatformAffinityFunction.cs @@ -21,13 +21,11 @@ namespace Apache.Ignite.Core.Impl.Cache.Affinity using System.Collections.Generic; using Apache.Ignite.Core.Cache.Affinity; using Apache.Ignite.Core.Cluster; - using Apache.Ignite.Core.Impl.Binary; - using Apache.Ignite.Core.Impl.Unmanaged; /// /// Affinity function that delegates to Java. /// - internal class PlatformAffinityFunction : PlatformTarget, IAffinityFunction + internal class PlatformAffinityFunction : PlatformTargetAdapter, IAffinityFunction { /** Opcodes. */ private enum Op @@ -41,8 +39,7 @@ private enum Op /// Initializes a new instance of the class. /// /// Target. - /// Marshaller. - public PlatformAffinityFunction(IUnmanagedTarget target, Marshaller marsh) : base(target, marsh) + public PlatformAffinityFunction(IPlatformTargetInternal target) : base(target) { // No-op. } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheAffinityImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheAffinityImpl.cs index f09a119700014..a2bba299712e6 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheAffinityImpl.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheAffinityImpl.cs @@ -19,19 +19,16 @@ namespace Apache.Ignite.Core.Impl.Cache { using System; using System.Collections.Generic; - using System.Diagnostics; using Apache.Ignite.Core.Cache; using Apache.Ignite.Core.Cluster; using Apache.Ignite.Core.Impl.Binary; using Apache.Ignite.Core.Impl.Binary.IO; using Apache.Ignite.Core.Impl.Common; - using Apache.Ignite.Core.Impl.Unmanaged; - using UU = Apache.Ignite.Core.Impl.Unmanaged.UnmanagedUtils; /// /// Cache affinity implementation. /// - internal class CacheAffinityImpl : PlatformTarget, ICacheAffinity + internal class CacheAffinityImpl : PlatformTargetAdapter, ICacheAffinity { /** */ private const int OpAffinityKey = 1; @@ -88,17 +85,12 @@ internal class CacheAffinityImpl : PlatformTarget, ICacheAffinity /// Initializes a new instance of the class. /// /// Target. - /// Marshaller. /// Keep binary flag. - /// Grid. - public CacheAffinityImpl(IUnmanagedTarget target, Marshaller marsh, bool keepBinary, - Ignite ignite) : base(target, marsh) + public CacheAffinityImpl(IPlatformTargetInternal target, bool keepBinary) : base(target) { _keepBinary = keepBinary; - Debug.Assert(ignite != null); - - _ignite = ignite; + _ignite = target.Marshaller.Ignite; } /** */ @@ -182,7 +174,7 @@ public int[] GetAllPartitions(IClusterNode n) { IgniteArgumentCheck.NotNull(keys, "keys"); - return DoOutInOp(OpMapKeysToNodes, w => WriteEnumerable(w, keys), + return DoOutInOp(OpMapKeysToNodes, w => w.WriteEnumerable(keys), reader => ReadDictionary(reader, ReadNode, r => (IList) r.ReadCollectionAsList())); } @@ -214,7 +206,7 @@ public IClusterNode MapPartitionToNode(int part) IgniteArgumentCheck.NotNull(parts, "parts"); return DoOutInOp(OpMapPartitionsToNodes, - w => WriteEnumerable(w, parts), + w => w.WriteEnumerable(parts), reader => ReadDictionary(reader, r => r.ReadInt(), ReadNode)); } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheEnumerator.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheEnumerator.cs index e2b8350aef63d..2860bb66c98f1 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheEnumerator.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheEnumerator.cs @@ -21,14 +21,12 @@ namespace Apache.Ignite.Core.Impl.Cache using System.Collections; using System.Collections.Generic; using Apache.Ignite.Core.Cache; - using Apache.Ignite.Core.Impl.Binary; using Apache.Ignite.Core.Impl.Binary.IO; - using Apache.Ignite.Core.Impl.Unmanaged; /// /// Real cache enumerator communicating with Java. /// - internal class CacheEnumerator : PlatformDisposableTarget, IEnumerator> + internal class CacheEnumerator : PlatformDisposableTargetAdapter, IEnumerator> { /** Operation: next value. */ private const int OpNext = 1; @@ -43,10 +41,8 @@ internal class CacheEnumerator : PlatformDisposableTarget, IEnumerator /// Target. - /// Marshaller. /// Keep binary flag. - public CacheEnumerator(IUnmanagedTarget target, Marshaller marsh, bool keepBinary) : - base(target, marsh) + public CacheEnumerator(IPlatformTargetInternal target, bool keepBinary) : base(target) { _keepBinary = keepBinary; } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs index e6b2408333456..5789c8fc14de3 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs @@ -20,7 +20,6 @@ namespace Apache.Ignite.Core.Impl.Cache using System; using System.Collections; using System.Collections.Generic; - using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Threading.Tasks; using Apache.Ignite.Core.Binary; @@ -38,13 +37,12 @@ namespace Apache.Ignite.Core.Impl.Cache using Apache.Ignite.Core.Impl.Cluster; using Apache.Ignite.Core.Impl.Common; using Apache.Ignite.Core.Impl.Transactions; - using Apache.Ignite.Core.Impl.Unmanaged; /// /// Native cache wrapper. /// [SuppressMessage("Microsoft.Design", "CA1001:TypesThatOwnDisposableFieldsShouldBeDisposable")] - internal class CacheImpl : PlatformTarget, ICache, ICacheInternal, ICacheLockInternal + internal class CacheImpl : PlatformTargetAdapter, ICache, ICacheInternal, ICacheLockInternal { /** Ignite instance. */ private readonly Ignite _ignite; @@ -64,31 +62,32 @@ internal class CacheImpl : PlatformTarget, ICache, ICacheInterna /** Transaction manager. */ private readonly CacheTransactionManager _txManager; + /** Pre-allocated delegate. */ + private readonly Func _readException; + /// /// Constructor. /// - /// Grid. /// Target. - /// Marshaller. /// Skip store flag. /// Keep binary flag. /// No-retries mode flag. /// Partition recover mode flag. - public CacheImpl(Ignite grid, IUnmanagedTarget target, Marshaller marsh, + public CacheImpl(IPlatformTargetInternal target, bool flagSkipStore, bool flagKeepBinary, bool flagNoRetries, bool flagPartitionRecover) - : base(target, marsh) + : base(target) { - Debug.Assert(grid != null); - - _ignite = grid; + _ignite = target.Marshaller.Ignite; _flagSkipStore = flagSkipStore; _flagKeepBinary = flagKeepBinary; _flagNoRetries = flagNoRetries; _flagPartitionRecover = flagPartitionRecover; _txManager = GetConfiguration().AtomicityMode == CacheAtomicityMode.Transactional - ? new CacheTransactionManager(grid.GetTransactions()) + ? new CacheTransactionManager(_ignite.GetTransactions()) : null; + + _readException = stream => ReadException(Marshaller.StartUnmarshal(stream)); } /** */ @@ -172,7 +171,7 @@ public bool IsEmpty() if (_flagSkipStore) return this; - return new CacheImpl(_ignite, DoOutOpObject((int) CacheOp.WithSkipStore), Marshaller, + return new CacheImpl(DoOutOpObject((int) CacheOp.WithSkipStore), true, _flagKeepBinary, true, _flagPartitionRecover); } @@ -196,7 +195,7 @@ public bool IsEmpty() return result; } - return new CacheImpl(_ignite, DoOutOpObject((int) CacheOp.WithKeepBinary), Marshaller, + return new CacheImpl(DoOutOpObject((int) CacheOp.WithKeepBinary), _flagSkipStore, true, _flagNoRetries, _flagPartitionRecover); } @@ -207,7 +206,7 @@ public bool IsEmpty() var cache0 = DoOutOpObject((int)CacheOp.WithExpiryPolicy, w => ExpiryPolicySerializer.WritePolicy(w, plc)); - return new CacheImpl(_ignite, cache0, Marshaller, _flagSkipStore, _flagKeepBinary, + return new CacheImpl(cache0, _flagSkipStore, _flagKeepBinary, _flagNoRetries, _flagPartitionRecover); } @@ -220,7 +219,7 @@ public bool IsKeepBinary /** */ public void LoadCache(ICacheEntryFilter p, params object[] args) { - DoOutInOpX((int) CacheOp.LoadCache, writer => WriteLoadCacheData(writer, p, args), ReadException); + DoOutInOpX((int) CacheOp.LoadCache, writer => WriteLoadCacheData(writer, p, args), _readException); } /** */ @@ -232,7 +231,7 @@ public Task LoadCacheAsync(ICacheEntryFilter p, params object[] args) /** */ public void LocalLoadCache(ICacheEntryFilter p, params object[] args) { - DoOutInOpX((int) CacheOp.LocLoadCache, writer => WriteLoadCacheData(writer, p, args), ReadException); + DoOutInOpX((int) CacheOp.LocLoadCache, writer => WriteLoadCacheData(writer, p, args), _readException); } /** */ @@ -281,7 +280,7 @@ public Task LoadAllAsync(IEnumerable keys, bool replaceExistingValues) return DoOutOpAsync(CacheOp.LoadAll, writer => { writer.WriteBoolean(replaceExistingValues); - WriteEnumerable(writer, keys); + writer.WriteEnumerable(keys); }); } @@ -306,7 +305,7 @@ public bool ContainsKeys(IEnumerable keys) { IgniteArgumentCheck.NotNull(keys, "keys"); - return DoOutOp(CacheOp.ContainsKeys, writer => WriteEnumerable(writer, keys)); + return DoOutOp(CacheOp.ContainsKeys, writer => writer.WriteEnumerable(keys)); } /** */ @@ -314,7 +313,7 @@ public Task ContainsKeysAsync(IEnumerable keys) { IgniteArgumentCheck.NotNull(keys, "keys"); - return DoOutOpAsync(CacheOp.ContainsKeysAsync, writer => WriteEnumerable(writer, keys)); + return DoOutOpAsync(CacheOp.ContainsKeysAsync, writer => writer.WriteEnumerable(keys)); } /** */ @@ -342,7 +341,7 @@ public bool TryLocalPeek(TK key, out TV value, params CachePeekMode[] modes) w.WriteInt(EncodePeekModes(modes)); }, (s, r) => r == True ? new CacheResult(Unmarshal(s)) : new CacheResult(), - ReadException); + _readException); value = res.Success ? res.Value : default(TV); @@ -375,7 +374,7 @@ public TV Get(TK key) throw GetKeyNotFoundException(); return Unmarshal(stream); - }, ReadException); + }, _readException); } /** */ @@ -418,9 +417,9 @@ public Task> TryGetAsync(TK key) IgniteArgumentCheck.NotNull(keys, "keys"); return DoOutInOpX((int) CacheOp.GetAll, - writer => WriteEnumerable(writer, keys), + writer => writer.WriteEnumerable(keys), (s, r) => r == True ? ReadGetAllDictionary(Marshaller.StartUnmarshal(s, _flagKeepBinary)) : null, - ReadException); + _readException); } /** */ @@ -428,7 +427,7 @@ public Task> TryGetAsync(TK key) { IgniteArgumentCheck.NotNull(keys, "keys"); - return DoOutOpAsync(CacheOp.GetAllAsync, w => WriteEnumerable(w, keys), r => ReadGetAllDictionary(r)); + return DoOutOpAsync(CacheOp.GetAllAsync, w => w.WriteEnumerable(keys), r => ReadGetAllDictionary(r)); } /** */ @@ -631,7 +630,7 @@ public void PutAll(IEnumerable> vals) StartTx(); - DoOutOp(CacheOp.PutAll, writer => WriteDictionary(writer, vals)); + DoOutOp(CacheOp.PutAll, writer => writer.WriteDictionary(vals)); } /** */ @@ -641,7 +640,7 @@ public Task PutAllAsync(IEnumerable> vals) StartTx(); - return DoOutOpAsync(CacheOp.PutAllAsync, writer => WriteDictionary(writer, vals)); + return DoOutOpAsync(CacheOp.PutAllAsync, writer => writer.WriteDictionary(vals)); } /** */ @@ -649,7 +648,7 @@ public void LocalEvict(IEnumerable keys) { IgniteArgumentCheck.NotNull(keys, "keys"); - DoOutOp(CacheOp.LocEvict, writer => WriteEnumerable(writer, keys)); + DoOutOp(CacheOp.LocEvict, writer => writer.WriteEnumerable(keys)); } /** */ @@ -685,7 +684,7 @@ public void ClearAll(IEnumerable keys) { IgniteArgumentCheck.NotNull(keys, "keys"); - DoOutOp(CacheOp.ClearAll, writer => WriteEnumerable(writer, keys)); + DoOutOp(CacheOp.ClearAll, writer => writer.WriteEnumerable(keys)); } /** */ @@ -693,7 +692,7 @@ public Task ClearAllAsync(IEnumerable keys) { IgniteArgumentCheck.NotNull(keys, "keys"); - return DoOutOpAsync(CacheOp.ClearAllAsync, writer => WriteEnumerable(writer, keys)); + return DoOutOpAsync(CacheOp.ClearAllAsync, writer => writer.WriteEnumerable(keys)); } /** */ @@ -709,7 +708,7 @@ public void LocalClearAll(IEnumerable keys) { IgniteArgumentCheck.NotNull(keys, "keys"); - DoOutOp(CacheOp.LocalClearAll, writer => WriteEnumerable(writer, keys)); + DoOutOp(CacheOp.LocalClearAll, writer => writer.WriteEnumerable(keys)); } /** */ @@ -761,7 +760,7 @@ public void RemoveAll(IEnumerable keys) StartTx(); - DoOutOp(CacheOp.RemoveAll, writer => WriteEnumerable(writer, keys)); + DoOutOp(CacheOp.RemoveAll, writer => writer.WriteEnumerable(keys)); } /** */ @@ -771,7 +770,7 @@ public Task RemoveAllAsync(IEnumerable keys) StartTx(); - return DoOutOpAsync(CacheOp.RemoveAllAsync, writer => WriteEnumerable(writer, keys)); + return DoOutOpAsync(CacheOp.RemoveAllAsync, writer => writer.WriteEnumerable(keys)); } /** */ @@ -843,7 +842,7 @@ private int Size0(bool loc, params CachePeekMode[] modes) writer.Write(holder); }, (input, res) => res == True ? Unmarshal(input) : default(TRes), - ReadException); + _readException); } /** */ @@ -891,10 +890,12 @@ private int Size0(bool loc, params CachePeekMode[] modes) return DoOutInOpX((int) CacheOp.InvokeAll, writer => { - WriteEnumerable(writer, keys); + writer.WriteEnumerable(keys); writer.Write(holder); }, - (input, res) => res == True ? ReadInvokeAllResults(Marshaller.StartUnmarshal(input, IsKeepBinary)): null, ReadException); + (input, res) => res == True + ? ReadInvokeAllResults(Marshaller.StartUnmarshal(input, IsKeepBinary)) + : null, _readException); } /** */ @@ -912,7 +913,7 @@ private int Size0(bool loc, params CachePeekMode[] modes) return DoOutOpAsync(CacheOp.InvokeAllAsync, writer => { - WriteEnumerable(writer, keys); + writer.WriteEnumerable(keys); writer.Write(holder); }, input => ReadInvokeAllResults(input)); @@ -931,7 +932,7 @@ private int Size0(bool loc, params CachePeekMode[] modes) }, (input, res) => res == True ? readFunc(Marshaller.StartUnmarshal(input)) - : default(T), ReadException); + : default(T), _readException); } /** */ @@ -940,7 +941,7 @@ public ICacheLock Lock(TK key) IgniteArgumentCheck.NotNull(key, "key"); return DoOutInOpX((int) CacheOp.Lock, w => w.Write(key), - (stream, res) => new CacheLock(stream.ReadInt(), this), ReadException); + (stream, res) => new CacheLock(stream.ReadInt(), this), _readException); } /** */ @@ -948,8 +949,8 @@ public ICacheLock LockAll(IEnumerable keys) { IgniteArgumentCheck.NotNull(keys, "keys"); - return DoOutInOpX((int) CacheOp.LockAll, w => WriteEnumerable(w, keys), - (stream, res) => new CacheLock(stream.ReadInt(), this), ReadException); + return DoOutInOpX((int) CacheOp.LockAll, w => w.WriteEnumerable(keys), + (stream, res) => new CacheLock(stream.ReadInt(), this), _readException); } /** */ @@ -1011,7 +1012,7 @@ public Task Rebalance() if (_flagNoRetries) return this; - return new CacheImpl(_ignite, DoOutOpObject((int) CacheOp.WithNoRetries), Marshaller, + return new CacheImpl(DoOutOpObject((int) CacheOp.WithNoRetries), _flagSkipStore, _flagKeepBinary, true, _flagPartitionRecover); } @@ -1021,7 +1022,7 @@ public Task Rebalance() if (_flagPartitionRecover) return this; - return new CacheImpl(_ignite, DoOutOpObject((int) CacheOp.WithPartitionRecover), Marshaller, + return new CacheImpl(DoOutOpObject((int) CacheOp.WithPartitionRecover), _flagSkipStore, _flagKeepBinary, _flagNoRetries, true); } @@ -1092,7 +1093,7 @@ public IQueryCursor QueryFields(SqlFieldsQuery qry, Func(cursor, Marshaller, _flagKeepBinary, readerFunc); + return new FieldsQueryCursor(cursor, _flagKeepBinary, readerFunc); } /** */ @@ -1102,7 +1103,7 @@ public IQueryCursor QueryFields(SqlFieldsQuery qry, Func qry.Write(writer, IsKeepBinary)); - return new QueryCursor(cursor, Marshaller, _flagKeepBinary); + return new QueryCursor(cursor, _flagKeepBinary); } /** */ @@ -1168,10 +1169,10 @@ IEnumerator IEnumerable.GetEnumerator() { var target = DoOutOpObject((int) CacheOp.LocIterator, (IBinaryStream s) => s.WriteInt(peekModes)); - return new CacheEnumerator(target, Marshaller, _flagKeepBinary); + return new CacheEnumerator(target, _flagKeepBinary); } - return new CacheEnumerator(DoOutOpObject((int) CacheOp.Iterator), Marshaller, _flagKeepBinary); + return new CacheEnumerator(DoOutOpObject((int) CacheOp.Iterator), _flagKeepBinary); } #endregion @@ -1227,14 +1228,6 @@ private static int EncodePeekModes(CachePeekMode[] modes) return results; } - /// - /// Reads the exception. - /// - private Exception ReadException(IBinaryStream stream) - { - return ReadException(Marshaller.StartUnmarshal(stream)); - } - /// /// Reads the exception, either in binary wrapper form, or as a pair of strings. /// @@ -1315,7 +1308,7 @@ private bool DoOutOp(CacheOp op, T1 x) return DoOutInOpX((int) op, w => { w.Write(x); - }, ReadException); + }, _readException); } /// @@ -1327,7 +1320,7 @@ private bool DoOutOp(CacheOp op, T1 x) { w.Write(x); w.Write(y); - }, ReadException); + }, _readException); } /// @@ -1340,7 +1333,7 @@ private bool DoOutOp(CacheOp op, T1 x) w.Write(x); w.Write(y); w.Write(z); - }, ReadException); + }, _readException); } /// @@ -1348,7 +1341,7 @@ private bool DoOutOp(CacheOp op, T1 x) /// private bool DoOutOp(CacheOp op, Action write) { - return DoOutInOpX((int) op, write, ReadException); + return DoOutInOpX((int) op, write, _readException); } /// @@ -1359,7 +1352,7 @@ private CacheResult DoOutInOpNullable(CacheOp cacheOp, TK x) return DoOutInOpX((int)cacheOp, w => w.Write(x), (stream, res) => res == True ? new CacheResult(Unmarshal(stream)) : new CacheResult(), - ReadException); + _readException); } /// @@ -1374,7 +1367,7 @@ private CacheResult DoOutInOpNullable(CacheOp cacheOp, TK x) w.Write(y); }, (stream, res) => res == True ? new CacheResult(Unmarshal(stream)) : new CacheResult(), - ReadException); + _readException); } /** */ diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/AbstractQueryCursor.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/AbstractQueryCursor.cs index 95c6a36f59985..8e4985ea1d6cf 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/AbstractQueryCursor.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/AbstractQueryCursor.cs @@ -23,13 +23,11 @@ namespace Apache.Ignite.Core.Impl.Cache.Query using Apache.Ignite.Core.Cache.Query; using Apache.Ignite.Core.Impl.Binary; using Apache.Ignite.Core.Impl.Binary.IO; - using Apache.Ignite.Core.Impl.Unmanaged; - using UU = Apache.Ignite.Core.Impl.Unmanaged.UnmanagedUtils; /// /// Abstract query cursor implementation. /// - internal abstract class AbstractQueryCursor : PlatformDisposableTarget, IQueryCursor, IEnumerator + internal abstract class AbstractQueryCursor : PlatformDisposableTargetAdapter, IQueryCursor, IEnumerator { /** */ private const int OpGetAll = 1; @@ -65,10 +63,8 @@ internal abstract class AbstractQueryCursor : PlatformDisposableTarget, IQuer /// Constructor. /// /// Target. - /// Marshaller. /// Keep binary flag. - protected AbstractQueryCursor(IUnmanagedTarget target, Marshaller marsh, bool keepBinary) : - base(target, marsh) + protected AbstractQueryCursor(IPlatformTargetInternal target, bool keepBinary) : base(target) { _keepBinary = keepBinary; } @@ -88,7 +84,7 @@ public IList GetAll() throw new InvalidOperationException("Failed to get all entries because GetAll() " + "method has already been called."); - var res = DoInOp>(OpGetAll, ConvertGetAll); + var res = DoInOp(OpGetAll, ConvertGetAll); _getAllCalled = true; @@ -216,7 +212,7 @@ protected override T1 Unmarshal(IBinaryStream stream) /// private void RequestBatch() { - _batch = DoInOp(OpGetBatch, ConvertGetBatch); + _batch = DoInOp(OpGetBatch, ConvertGetBatch); _batchPos = 0; } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/Continuous/ContinuousQueryHandleImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/Continuous/ContinuousQueryHandleImpl.cs index 6139d8be51fdd..ff5c43404aca6 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/Continuous/ContinuousQueryHandleImpl.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/Continuous/ContinuousQueryHandleImpl.cs @@ -28,8 +28,6 @@ namespace Apache.Ignite.Core.Impl.Cache.Query.Continuous using Apache.Ignite.Core.Impl.Binary.IO; using Apache.Ignite.Core.Impl.Common; using Apache.Ignite.Core.Impl.Resource; - using Apache.Ignite.Core.Impl.Unmanaged; - using UU = Apache.Ignite.Core.Impl.Unmanaged.UnmanagedUtils; using CQU = ContinuousQueryUtils; /// @@ -67,7 +65,7 @@ internal class ContinuousQueryHandleImpl : IContinuousQueryHandleImpl, I private readonly long _hnd; /** Native query. */ - private readonly IUnmanagedTarget _nativeQry; + private readonly IPlatformTargetInternal _nativeQry; /** Initial query cursor. */ private volatile IQueryCursor> _initialQueryCursor; @@ -84,7 +82,7 @@ internal class ContinuousQueryHandleImpl : IContinuousQueryHandleImpl, I /// The initialization callback. /// The initial query. public ContinuousQueryHandleImpl(ContinuousQuery qry, Marshaller marsh, bool keepBinary, - Func, IUnmanagedTarget> createTargetCb, QueryBase initialQry) + Func, IPlatformTargetInternal> createTargetCb, QueryBase initialQry) { _marsh = marsh; _keepBinary = keepBinary; @@ -138,10 +136,10 @@ internal class ContinuousQueryHandleImpl : IContinuousQueryHandleImpl, I }); // 4. Initial query. - var nativeInitialQryCur = UU.TargetOutObject(_nativeQry, 0); + var nativeInitialQryCur = _nativeQry.OutObjectInternal(0); _initialQueryCursor = nativeInitialQryCur == null ? null - : new QueryCursor(nativeInitialQryCur, _marsh, _keepBinary); + : new QueryCursor(nativeInitialQryCur, _keepBinary); } catch (Exception) { @@ -225,7 +223,7 @@ public void Dispose() try { - UU.TargetInLongOutLong(_nativeQry, 0, 0); + _nativeQry.InLongOutLong(0, 0); } finally { diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/FieldsQueryCursor.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/FieldsQueryCursor.cs index d928418a185a9..9d021dc41c300 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/FieldsQueryCursor.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/FieldsQueryCursor.cs @@ -22,7 +22,6 @@ namespace Apache.Ignite.Core.Impl.Cache.Query using System.Diagnostics.CodeAnalysis; using Apache.Ignite.Core.Binary; using Apache.Ignite.Core.Impl.Binary; - using Apache.Ignite.Core.Impl.Unmanaged; /// /// Cursor for entry-based queries. @@ -36,12 +35,11 @@ internal class FieldsQueryCursor : AbstractQueryCursor /// Constructor. /// /// Target. - /// Marshaler. /// Keep poratble flag. /// The reader function. - public FieldsQueryCursor(IUnmanagedTarget target, Marshaller marsh, bool keepBinary, + public FieldsQueryCursor(IPlatformTargetInternal target, bool keepBinary, Func readerFunc) - : base(target, marsh, keepBinary) + : base(target, keepBinary) { Debug.Assert(readerFunc != null); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/QueryCursor.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/QueryCursor.cs index 5a469157ac912..bc3cdb6896cc8 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/QueryCursor.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/QueryCursor.cs @@ -20,7 +20,6 @@ namespace Apache.Ignite.Core.Impl.Cache.Query using System.Diagnostics.CodeAnalysis; using Apache.Ignite.Core.Cache; using Apache.Ignite.Core.Impl.Binary; - using Apache.Ignite.Core.Impl.Unmanaged; /// /// Cursor for entry-based queries. @@ -31,10 +30,8 @@ internal class QueryCursor : AbstractQueryCursor> /// Constructor. /// /// Target. - /// Marshaler. /// Keep poratble flag. - public QueryCursor(IUnmanagedTarget target, Marshaller marsh, - bool keepBinary) : base(target, marsh, keepBinary) + public QueryCursor(IPlatformTargetInternal target, bool keepBinary) : base(target, keepBinary) { // No-op. } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cluster/ClusterGroupImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cluster/ClusterGroupImpl.cs index 30afe5785b9ff..678fb035b0521 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cluster/ClusterGroupImpl.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cluster/ClusterGroupImpl.cs @@ -37,16 +37,14 @@ namespace Apache.Ignite.Core.Impl.Cluster using Apache.Ignite.Core.Impl.Messaging; using Apache.Ignite.Core.Impl.PersistentStore; using Apache.Ignite.Core.Impl.Services; - using Apache.Ignite.Core.Impl.Unmanaged; using Apache.Ignite.Core.Messaging; using Apache.Ignite.Core.PersistentStore; using Apache.Ignite.Core.Services; - using UU = Apache.Ignite.Core.Impl.Unmanaged.UnmanagedUtils; /// /// Ignite projection implementation. /// - internal class ClusterGroupImpl : PlatformTarget, IClusterGroup + internal class ClusterGroupImpl : PlatformTargetAdapter, IClusterGroup { /** Attribute: platform. */ private const string AttrPlatform = "org.apache.ignite.platform"; @@ -175,13 +173,12 @@ internal class ClusterGroupImpl : PlatformTarget, IClusterGroup /// Constructor. /// /// Target. - /// Grid. /// Predicate. [SuppressMessage("Microsoft.Performance", "CA1805:DoNotInitializeUnnecessarily")] - public ClusterGroupImpl(IUnmanagedTarget target, Ignite ignite, Func pred) - : base(target, ignite.Marshaller) + public ClusterGroupImpl(IPlatformTargetInternal target, Func pred) + : base(target) { - _ignite = ignite; + _ignite = target.Marshaller.Ignite; _pred = pred; _comp = new Lazy(() => CreateCompute()); @@ -207,7 +204,7 @@ public ICompute GetCompute() /// private ICompute CreateCompute() { - return new Compute(new ComputeImpl(DoOutOpObject(OpGetCompute), Marshaller, this, false)); + return new Compute(new ComputeImpl(DoOutOpObject(OpGetCompute), this, false)); } /** */ @@ -252,10 +249,7 @@ private IClusterGroup ForNodeIds0(IEnumerable items, Func func) { Debug.Assert(items != null); - IUnmanagedTarget prj = DoOutOpObject(OpForNodeIds, writer => - { - WriteEnumerable(writer, items, func); - }); + var prj = DoOutOpObject(OpForNodeIds, writer => writer.WriteEnumerable(items, func)); return GetClusterGroup(prj); } @@ -265,7 +259,7 @@ public IClusterGroup ForPredicate(Func p) { var newPred = _pred == null ? p : node => _pred(node) && p(node); - return new ClusterGroupImpl(Target, _ignite, newPred); + return new ClusterGroupImpl(Target, newPred); } /** */ @@ -278,7 +272,7 @@ public IClusterGroup ForAttribute(string name, string val) writer.WriteString(name); writer.WriteString(val); }; - IUnmanagedTarget prj = DoOutOpObject(OpForAttribute, action); + var prj = DoOutOpObject(OpForAttribute, action); return GetClusterGroup(prj); } @@ -293,7 +287,7 @@ public IClusterGroup ForAttribute(string name, string val) /// private IClusterGroup ForCacheNodes(string name, int op) { - IUnmanagedTarget prj = DoOutOpObject(op, writer => + var prj = DoOutOpObject(op, writer => { writer.WriteString(name); }); @@ -336,7 +330,7 @@ public IClusterGroup ForHost(IClusterNode node) { IgniteArgumentCheck.NotNull(node, "node"); - IUnmanagedTarget prj = DoOutOpObject(OpForHost, writer => + var prj = DoOutOpObject(OpForHost, writer => { writer.WriteGuid(node.Id); }); @@ -404,15 +398,14 @@ public IClusterMetrics GetMetrics() return reader.ReadBoolean() ? new ClusterMetricsImpl(reader) : null; }); } - return DoOutInOp(OpMetricsFiltered, writer => - { - WriteEnumerable(writer, GetNodes().Select(node => node.Id)); - }, stream => - { - IBinaryRawReader reader = Marshaller.StartUnmarshal(stream, false); + return DoOutInOp(OpMetricsFiltered, + writer => writer.WriteEnumerable(GetNodes().Select(node => node.Id)), + stream => + { + IBinaryRawReader reader = Marshaller.StartUnmarshal(stream, false); - return reader.ReadBoolean() ? new ClusterMetricsImpl(reader) : null; - }); + return reader.ReadBoolean() ? new ClusterMetricsImpl(reader) : null; + }); } /** */ @@ -426,7 +419,7 @@ public IMessaging GetMessaging() /// private IMessaging CreateMessaging() { - return new Messaging(DoOutOpObject(OpGetMessaging), Marshaller, this); + return new Messaging(DoOutOpObject(OpGetMessaging), this); } /** */ @@ -440,7 +433,7 @@ public IEvents GetEvents() /// private IEvents CreateEvents() { - return new Events(DoOutOpObject(OpGetEvents), Marshaller, this); + return new Events(DoOutOpObject(OpGetEvents), this); } /** */ @@ -454,7 +447,7 @@ public IServices GetServices() /// private IServices CreateServices() { - return new Services(DoOutOpObject(OpGetServices), Marshaller, this, false, false); + return new Services(DoOutOpObject(OpGetServices), this, false, false); } /// @@ -665,9 +658,9 @@ public IPersistentStoreMetrics GetPersistentStoreMetrics() /// /// Native projection. /// New cluster group. - private IClusterGroup GetClusterGroup(IUnmanagedTarget prj) + private IClusterGroup GetClusterGroup(IPlatformTargetInternal prj) { - return new ClusterGroupImpl(prj, _ignite, _pred); + return new ClusterGroupImpl(prj, _pred); } /// @@ -678,29 +671,30 @@ private IList RefreshNodes() { long oldTopVer = Interlocked.Read(ref _topVer); - List newNodes = null; - - DoOutInOp(OpNodes, writer => + var res = Target.InStreamOutStream(OpNodes, writer => { writer.WriteLong(oldTopVer); - }, input => + }, reader => { - BinaryReader reader = Marshaller.StartUnmarshal(input); - if (reader.ReadBoolean()) { // Topology has been updated. long newTopVer = reader.ReadLong(); + var newNodes = IgniteUtils.ReadNodes((BinaryReader) reader, _pred); - newNodes = IgniteUtils.ReadNodes(reader, _pred); - - UpdateTopology(newTopVer, newNodes); + return Tuple.Create(newTopVer, newNodes); } + + return null; }); - if (newNodes != null) - return newNodes; - + if (res != null) + { + UpdateTopology(res.Item1, res.Item2); + + return res.Item2; + } + // No topology changes. Debug.Assert(_nodes != null, "At least one topology update should have occurred."); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/DelegateTypeDescriptor.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/DelegateTypeDescriptor.cs index 4cd0678db6d21..cc12caaad2583 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/DelegateTypeDescriptor.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/DelegateTypeDescriptor.cs @@ -28,7 +28,6 @@ namespace Apache.Ignite.Core.Impl.Common using Apache.Ignite.Core.Impl.Cache; using Apache.Ignite.Core.Impl.Cache.Query.Continuous; using Apache.Ignite.Core.Impl.Datastream; - using Apache.Ignite.Core.Impl.Unmanaged; using Apache.Ignite.Core.Messaging; /// @@ -66,7 +65,7 @@ internal class DelegateTypeDescriptor private readonly Action _computeJobCancel; /** */ - private readonly Action _streamReceiver; + private readonly Action _streamReceiver; /** */ private readonly Func _streamTransformerCtor; @@ -163,7 +162,7 @@ public static void GetComputeJob(Type type, out Func execute, ou /// /// Type. /// Precompiled invocator delegate. - public static Action GetStreamReceiver(Type type) + public static Action GetStreamReceiver(Type type) { return Get(type)._streamReceiver; } @@ -313,12 +312,12 @@ private DelegateTypeDescriptor(Type type) .MakeGenericMethod(iface.GetGenericArguments()); _streamReceiver = DelegateConverter - .CompileFunc>( + .CompileFunc>( typeof (StreamReceiverHolder), method, new[] { - iface, typeof (Ignite), typeof (IUnmanagedTarget), typeof (IBinaryStream), + iface, typeof (Ignite), typeof (IPlatformTargetInternal), typeof (IBinaryStream), typeof (bool) }, new[] {true, false, false, false, false, false}); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/Listenable.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/Listenable.cs index 6da98ab74ae0c..8566d0bd3ea25 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/Listenable.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/Listenable.cs @@ -17,13 +17,10 @@ namespace Apache.Ignite.Core.Impl.Common { - using Apache.Ignite.Core.Impl.Binary; - using Apache.Ignite.Core.Impl.Unmanaged; - /// /// Platform listenable. /// - internal class Listenable : PlatformTarget + internal class Listenable : PlatformTargetAdapter { /** */ private const int OpCancel = 1; @@ -32,8 +29,7 @@ internal class Listenable : PlatformTarget /// Initializes a new instance of the class. /// /// Target. - /// Marshaller. - public Listenable(IUnmanagedTarget target, Marshaller marsh) : base(target, marsh) + public Listenable(IPlatformTargetInternal target) : base(target) { // No-op. } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeImpl.cs index cace7b2211770..06f9ad413fd34 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeImpl.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Compute/ComputeImpl.cs @@ -34,13 +34,12 @@ namespace Apache.Ignite.Core.Impl.Compute using Apache.Ignite.Core.Impl.Cluster; using Apache.Ignite.Core.Impl.Common; using Apache.Ignite.Core.Impl.Compute.Closure; - using Apache.Ignite.Core.Impl.Unmanaged; /// /// Compute implementation. /// [SuppressMessage("Microsoft.Design", "CA1001:TypesThatOwnDisposableFieldsShouldBeDisposable")] - internal class ComputeImpl : PlatformTarget + internal class ComputeImpl : PlatformTargetAdapter { /** */ private const int OpAffinity = 1; @@ -76,11 +75,10 @@ internal class ComputeImpl : PlatformTarget /// Constructor. /// /// Target. - /// Marshaller. /// Projection. /// Binary flag. - public ComputeImpl(IUnmanagedTarget target, Marshaller marsh, ClusterGroupImpl prj, bool keepBinary) - : base(target, marsh) + public ComputeImpl(IPlatformTargetInternal target, ClusterGroupImpl prj, bool keepBinary) + : base(target) { _prj = prj; @@ -194,7 +192,7 @@ public Future ExecuteJavaTaskAsync(string taskName, obje var future = holder.Future; - future.SetTarget(new Listenable(futTarget, Marshaller)); + future.SetTarget(new Listenable(futTarget)); return future; } @@ -551,7 +549,7 @@ protected override T Unmarshal(IBinaryStream stream) writeAction(writer); }); - holder.Future.SetTarget(new Listenable(futTarget, Marshaller)); + holder.Future.SetTarget(new Listenable(futTarget)); } catch (Exception e) { diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/DataStructures/AtomicLong.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/DataStructures/AtomicLong.cs index 0c4bf84b17ceb..f797408f92c93 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/DataStructures/AtomicLong.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/DataStructures/AtomicLong.cs @@ -19,16 +19,12 @@ namespace Apache.Ignite.Core.Impl.DataStructures { using System.Diagnostics; using Apache.Ignite.Core.DataStructures; - using Apache.Ignite.Core.Impl.Binary; using Apache.Ignite.Core.Impl.Binary.IO; - using Apache.Ignite.Core.Impl.Unmanaged; - - using UU = Apache.Ignite.Core.Impl.Unmanaged.UnmanagedUtils; /// /// Atomic long wrapper. /// - internal sealed class AtomicLong : PlatformTarget, IAtomicLong + internal sealed class AtomicLong : PlatformTargetAdapter, IAtomicLong { /** */ private readonly string _name; @@ -50,9 +46,8 @@ private enum Op /// Initializes a new instance of the class. /// /// The target. - /// The marshaller. /// The name. - public AtomicLong(IUnmanagedTarget target, Marshaller marsh, string name) : base(target, marsh) + public AtomicLong(IPlatformTargetInternal target, string name) : base(target) { Debug.Assert(!string.IsNullOrEmpty(name)); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/DataStructures/AtomicReference.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/DataStructures/AtomicReference.cs index 4ca4b249b3b2a..76515a29f43b9 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/DataStructures/AtomicReference.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/DataStructures/AtomicReference.cs @@ -19,13 +19,11 @@ namespace Apache.Ignite.Core.Impl.DataStructures { using System.Diagnostics; using Apache.Ignite.Core.DataStructures; - using Apache.Ignite.Core.Impl.Binary; - using Apache.Ignite.Core.Impl.Unmanaged; /// /// Atomic reference. /// - internal class AtomicReference : PlatformTarget, IAtomicReference + internal class AtomicReference : PlatformTargetAdapter, IAtomicReference { /** Opcodes. */ private enum Op @@ -41,8 +39,8 @@ private enum Op private readonly string _name; /** */ - public AtomicReference(IUnmanagedTarget target, Marshaller marsh, string name) - : base(target, marsh) + public AtomicReference(IPlatformTargetInternal target, string name) + : base(target) { Debug.Assert(!string.IsNullOrEmpty(name)); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/DataStructures/AtomicSequence.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/DataStructures/AtomicSequence.cs index f7fc6b7fb2e44..dd079ef4ae256 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/DataStructures/AtomicSequence.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/DataStructures/AtomicSequence.cs @@ -19,13 +19,11 @@ namespace Apache.Ignite.Core.Impl.DataStructures { using System.Diagnostics; using Apache.Ignite.Core.DataStructures; - using Apache.Ignite.Core.Impl.Binary; - using Apache.Ignite.Core.Impl.Unmanaged; /// /// Atomic long wrapper. /// - internal sealed class AtomicSequence: PlatformTarget, IAtomicSequence + internal sealed class AtomicSequence: PlatformTargetAdapter, IAtomicSequence { /** */ private readonly string _name; @@ -46,10 +44,9 @@ private enum Op /// Initializes a new instance of the class. /// /// The target. - /// The marshaller. /// The name. - public AtomicSequence(IUnmanagedTarget target, Marshaller marsh, string name) - : base(target, marsh) + public AtomicSequence(IPlatformTargetInternal target, string name) + : base(target) { Debug.Assert(!string.IsNullOrEmpty(name)); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerImpl.cs index 96e58d4cccc35..fb2df0167fb17 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerImpl.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerImpl.cs @@ -26,8 +26,6 @@ namespace Apache.Ignite.Core.Impl.Datastream using Apache.Ignite.Core.Datastream; using Apache.Ignite.Core.Impl.Binary; using Apache.Ignite.Core.Impl.Common; - using Apache.Ignite.Core.Impl.Unmanaged; - using UU = Apache.Ignite.Core.Impl.Unmanaged.UnmanagedUtils; /// /// Data streamer internal interface to get rid of generics. @@ -45,7 +43,7 @@ internal interface IDataStreamer /// /// Data streamer implementation. /// - internal class DataStreamerImpl : PlatformDisposableTarget, IDataStreamer, IDataStreamer + internal class DataStreamerImpl : PlatformDisposableTargetAdapter, IDataStreamer, IDataStreamer { #pragma warning disable 0420 @@ -141,8 +139,8 @@ internal class DataStreamerImpl : PlatformDisposableTarget, IDataStreame /// Marshaller. /// Cache name. /// Binary flag. - public DataStreamerImpl(IUnmanagedTarget target, Marshaller marsh, string cacheName, bool keepBinary) - : base(target, marsh) + public DataStreamerImpl(IPlatformTargetInternal target, Marshaller marsh, string cacheName, bool keepBinary) + : base(target) { _cacheName = cacheName; _keepBinary = keepBinary; diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/StreamReceiverHolder.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/StreamReceiverHolder.cs index c91334d1dce73..b717d1485782f 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/StreamReceiverHolder.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/StreamReceiverHolder.cs @@ -27,7 +27,6 @@ namespace Apache.Ignite.Core.Impl.Datastream using Apache.Ignite.Core.Impl.Binary.IO; using Apache.Ignite.Core.Impl.Cache; using Apache.Ignite.Core.Impl.Common; - using Apache.Ignite.Core.Impl.Unmanaged; /// /// Binary wrapper for . @@ -44,7 +43,7 @@ internal class StreamReceiverHolder : IBinaryWriteAware private readonly object _rcv; /** Invoker delegate. */ - private readonly Action _invoke; + private readonly Action _invoke; /// /// Initializes a new instance of the class. @@ -77,7 +76,7 @@ public StreamReceiverHolder(IBinaryRawReader reader) /// Receiver. /// Invoke delegate. public StreamReceiverHolder(object rcv, - Action invoke) + Action invoke) { Debug.Assert(rcv != null); Debug.Assert(invoke != null); @@ -109,7 +108,7 @@ public void WriteBinary(IBinaryWriter writer) /// Cache. /// Stream. /// Binary flag. - public void Receive(Ignite grid, IUnmanagedTarget cache, IBinaryStream stream, bool keepBinary) + public void Receive(Ignite grid, IPlatformTargetInternal cache, IBinaryStream stream, bool keepBinary) { Debug.Assert(grid != null); Debug.Assert(cache != null); @@ -126,8 +125,8 @@ public void Receive(Ignite grid, IUnmanagedTarget cache, IBinaryStream stream, b /// Cache. /// Stream. /// Binary flag. - public static void InvokeReceiver(IStreamReceiver receiver, Ignite grid, IUnmanagedTarget cache, - IBinaryStream stream, bool keepBinary) + public static void InvokeReceiver(IStreamReceiver receiver, Ignite grid, + IPlatformTargetInternal cache, IBinaryStream stream, bool keepBinary) { var reader = grid.Marshaller.StartUnmarshal(stream, keepBinary); @@ -138,7 +137,7 @@ public void Receive(Ignite grid, IUnmanagedTarget cache, IBinaryStream stream, b for (var i = 0; i < size; i++) entries.Add(new CacheEntry(reader.ReadObject(), reader.ReadObject())); - receiver.Receive(grid.GetCache(cache, keepBinary), entries); + receiver.Receive(Ignite.GetCache(cache, keepBinary), entries); } } } \ No newline at end of file diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Events/Events.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Events/Events.cs index eb454d62f3b12..3c7363e7b35b5 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Events/Events.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Events/Events.cs @@ -30,13 +30,11 @@ namespace Apache.Ignite.Core.Impl.Events using Apache.Ignite.Core.Impl.Binary.IO; using Apache.Ignite.Core.Impl.Common; using Apache.Ignite.Core.Impl.Handle; - using Apache.Ignite.Core.Impl.Unmanaged; - using UU = Apache.Ignite.Core.Impl.Unmanaged.UnmanagedUtils; /// /// Ignite events. /// - internal sealed class Events : PlatformTarget, IEvents + internal sealed class Events : PlatformTargetAdapter, IEvents { /// /// Opcodes. @@ -66,15 +64,14 @@ private enum Op /** Cluster group. */ private readonly IClusterGroup _clusterGroup; - + /// /// Initializes a new instance of the class. /// /// Target. - /// Marshaller. /// Cluster group. - public Events(IUnmanagedTarget target, Marshaller marsh, IClusterGroup clusterGroup) - : base(target, marsh) + public Events(IPlatformTargetInternal target, IClusterGroup clusterGroup) + : base(target) { Debug.Assert(clusterGroup != null); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/IPlatformTargetInternal.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/IPlatformTargetInternal.cs new file mode 100644 index 0000000000000..23174b419156c --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/IPlatformTargetInternal.cs @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Impl +{ + using System; + using Apache.Ignite.Core.Impl.Binary; + using Apache.Ignite.Core.Impl.Binary.IO; + using Apache.Ignite.Core.Interop; + + /// + /// Extended platform target interface with methods that operate on internal entities (streams and targets). + /// + internal interface IPlatformTargetInternal : IPlatformTarget, IDisposable + { + /// + /// Gets the marshaller. + /// + Marshaller Marshaller { get; } + + /// + /// Performs InStreamOutLong operation. + /// + /// Operation type code. + /// Write action. + /// Result. + long InStreamOutLong(int type, Action writeAction); + + /// + /// Performs InStreamOutLong operation with stream reuse. + /// + /// Operation type code. + /// Write action. + /// Read action. + /// Error action. + /// + /// Result. + /// + T InStreamOutLong(int type, Action writeAction, Func readAction, + Func readErrorAction); + + /// + /// Performs InStreamOutStream operation. + /// + /// Result type. + /// Operation type code. + /// Write action. + /// Read action. + /// Result. + T InStreamOutStream(int type, Action writeAction, Func readAction); + + /// + /// Performs InStreamOutObject operation. + /// + /// Operation type code. + /// Write action. + /// Result. + IPlatformTargetInternal InStreamOutObject(int type, Action writeAction); + + /// + /// Performs InObjectStreamOutObjectStream operation. + /// + /// Result type. + /// Operation type code. + /// Target argument. + /// Write action. + /// Read action. + /// Result. + T InObjectStreamOutObjectStream(int type, Action writeAction, + Func readAction, IPlatformTargetInternal arg); + + /// + /// Performs OutStream operation. + /// + /// Result type. + /// Operation type code. + /// Read action. + /// Result. + T OutStream(int type, Func readAction); + + /// + /// Performs the OutObject operation. + /// + /// Operation type code. + /// Result. + IPlatformTargetInternal OutObjectInternal(int type); + } +} \ No newline at end of file diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs index 715776e2c2beb..aae6ce73814c6 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs @@ -28,6 +28,7 @@ namespace Apache.Ignite.Core.Impl using Apache.Ignite.Core.Cache; using Apache.Ignite.Core.Cache.Configuration; using Apache.Ignite.Core.Cluster; + using Apache.Ignite.Core.Common; using Apache.Ignite.Core.Compute; using Apache.Ignite.Core.Datastream; using Apache.Ignite.Core.DataStructures; @@ -54,7 +55,7 @@ namespace Apache.Ignite.Core.Impl /// /// Native Ignite wrapper. /// - internal class Ignite : PlatformTarget, IIgnite, ICluster + internal class Ignite : PlatformTargetAdapter, IIgnite, ICluster { /// /// Operation codes for PlatformProcessorImpl calls. @@ -92,7 +93,7 @@ private enum Op private readonly string _name; /** Unmanaged node. */ - private readonly IUnmanagedTarget _proc; + private readonly IPlatformTargetInternal _proc; /** Marshaller. */ private readonly Marshaller _marsh; @@ -138,8 +139,8 @@ private enum Op /// Marshaller. /// Lifecycle beans. /// Callbacks. - public Ignite(IgniteConfiguration cfg, string name, IUnmanagedTarget proc, Marshaller marsh, - IList lifecycleHandlers, UnmanagedCallbacks cbs) : base(proc, marsh) + public Ignite(IgniteConfiguration cfg, string name, IPlatformTargetInternal proc, Marshaller marsh, + IList lifecycleHandlers, UnmanagedCallbacks cbs) : base(proc) { Debug.Assert(cfg != null); Debug.Assert(proc != null); @@ -156,17 +157,17 @@ private enum Op marsh.Ignite = this; - _prj = new ClusterGroupImpl(DoOutOpObject((int) Op.GetClusterGroup), this, null); + _prj = new ClusterGroupImpl(Target.OutObjectInternal((int) Op.GetClusterGroup), null); _binary = new Binary.Binary(marsh); - _binaryProc = new BinaryProcessor(DoOutOpObject((int) Op.GetBinaryProcessor), marsh); + _binaryProc = new BinaryProcessor(DoOutOpObject((int) Op.GetBinaryProcessor)); cbs.Initialize(this); // Grid is not completely started here, can't initialize interop transactions right away. _transactions = new Lazy( - () => new TransactionsImpl(DoOutOpObject((int) Op.GetTransactions), marsh, GetLocalNode().Id)); + () => new TransactionsImpl(DoOutOpObject((int) Op.GetTransactions), GetLocalNode().Id)); // Set reconnected task to completed state for convenience. _clientReconnectTaskCompletionSource.SetResult(false); @@ -380,7 +381,14 @@ public void Dispose() /// Cancel flag. internal unsafe void Stop(bool cancel) { - UU.IgnitionStop(_proc.Context, Name, cancel); + var jniTarget = _proc as PlatformJniTarget; + + if (jniTarget == null) + { + throw new IgniteException("Ignition.Stop is not supported in thin client."); + } + + UU.IgnitionStop(jniTarget.Target.Context, Name, cancel); _cbs.Cleanup(); } @@ -507,9 +515,9 @@ public void DestroyCache(string name) /// /// New instance of cache wrapping specified native cache. /// - public ICache GetCache(IUnmanagedTarget nativeCache, bool keepBinary = false) + public static ICache GetCache(IPlatformTargetInternal nativeCache, bool keepBinary = false) { - return new CacheImpl(this, nativeCache, _marsh, false, keepBinary, false, false); + return new CacheImpl(nativeCache, false, keepBinary, false, false); } /** */ @@ -585,7 +593,7 @@ public ICacheAffinity GetAffinity(string cacheName) var aff = DoOutOpObject((int) Op.GetAffinity, w => w.WriteString(cacheName)); - return new CacheAffinityImpl(aff, _marsh, false, this); + return new CacheAffinityImpl(aff, false); } /** */ @@ -627,7 +635,7 @@ public IAtomicLong GetAtomicLong(string name, long initialValue, bool create) if (nativeLong == null) return null; - return new AtomicLong(nativeLong, Marshaller, name); + return new AtomicLong(nativeLong, name); } /** */ @@ -645,7 +653,7 @@ public IAtomicSequence GetAtomicSequence(string name, long initialValue, bool cr if (nativeSeq == null) return null; - return new AtomicSequence(nativeSeq, Marshaller, name); + return new AtomicSequence(nativeSeq, name); } /** */ @@ -660,7 +668,7 @@ public IAtomicReference GetAtomicReference(string name, T initialValue, bo w.WriteBoolean(create); }); - return refTarget == null ? null : new AtomicReference(refTarget, Marshaller, name); + return refTarget == null ? null : new AtomicReference(refTarget, name); } /** */ @@ -685,7 +693,7 @@ public IgniteConfiguration GetConfiguration() /** */ public ICollection GetCacheNames() { - return OutStream((int) Op.GetCacheNames, r => + return Target.OutStream((int) Op.GetCacheNames, r => { var res = new string[r.ReadInt()]; @@ -848,7 +856,7 @@ public ClusterNodeImpl GetNode(Guid? id) /// /// Gets the interop processor. /// - internal IUnmanagedTarget InteropProcessor + internal IPlatformTargetInternal InteropProcessor { get { return _proc; } } @@ -891,7 +899,7 @@ internal PluginProcessor PluginProcessor /// internal void ProcessorReleaseStart() { - InLongOutLong((int) Op.ReleaseStart, 0); + Target.InLongOutLong((int) Op.ReleaseStart, 0); } /// @@ -899,7 +907,7 @@ internal void ProcessorReleaseStart() /// internal bool LoggerIsLevelEnabled(LogLevel logLevel) { - return InLongOutLong((int) Op.LoggerIsLevelEnabled, (long) logLevel) == True; + return Target.InLongOutLong((int) Op.LoggerIsLevelEnabled, (long) logLevel) == True; } /// @@ -907,7 +915,7 @@ internal bool LoggerIsLevelEnabled(LogLevel logLevel) /// internal void LoggerLog(LogLevel level, string msg, string category, string err) { - InStreamOutLong((int) Op.LoggerLog, w => + Target.InStreamOutLong((int) Op.LoggerLog, w => { w.WriteInt((int) level); w.WriteString(msg); @@ -921,7 +929,7 @@ internal void LoggerLog(LogLevel level, string msg, string category, string err) /// internal IPlatformTarget GetExtension(int id) { - return InStreamOutObject((int) Op.GetExtension, w => w.WriteInt(id)); + return ((IPlatformTarget) Target).InStreamOutObject((int) Op.GetExtension, w => w.WriteInt(id)); } } } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Messaging/Messaging.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Messaging/Messaging.cs index 1b43438069beb..e17bcbfaeda0a 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Messaging/Messaging.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Messaging/Messaging.cs @@ -29,13 +29,12 @@ namespace Apache.Ignite.Core.Impl.Messaging using Apache.Ignite.Core.Impl.Collections; using Apache.Ignite.Core.Impl.Common; using Apache.Ignite.Core.Impl.Resource; - using Apache.Ignite.Core.Impl.Unmanaged; using Apache.Ignite.Core.Messaging; /// /// Messaging functionality. /// - internal class Messaging : PlatformTarget, IMessaging + internal class Messaging : PlatformTargetAdapter, IMessaging { /// /// Opcodes. @@ -67,10 +66,9 @@ private enum Op /// Initializes a new instance of the class. /// /// Target. - /// Marshaller. /// Cluster group. - public Messaging(IUnmanagedTarget target, Marshaller marsh, IClusterGroup prj) - : base(target, marsh) + public Messaging(IPlatformTargetInternal target, IClusterGroup prj) + : base(target) { Debug.Assert(prj != null); @@ -102,7 +100,7 @@ public void SendAll(IEnumerable messages, object topic = null) { writer.Write(topic); - WriteEnumerable(writer, messages.OfType()); + writer.WriteEnumerable(messages.OfType()); }); } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/PlatformDisposableTargetAdapter.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/PlatformDisposableTargetAdapter.cs new file mode 100644 index 0000000000000..f884c40c6bc56 --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/PlatformDisposableTargetAdapter.cs @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Impl +{ + using System; + + /// + /// PlatformTargetAdapter with IDisposable pattern. + /// + internal abstract class PlatformDisposableTargetAdapter : PlatformTargetAdapter, IDisposable + { + /** Disposed flag. */ + private volatile bool _disposed; + + /// + /// Constructor. + /// + /// Target. + protected PlatformDisposableTargetAdapter(IPlatformTargetInternal target) : base(target) + { + // No-op. + } + + /** */ + public void Dispose() + { + lock (this) + { + if (_disposed) + return; + + Dispose(true); + + GC.SuppressFinalize(this); + + _disposed = true; + } + } + + /// + /// Releases unmanaged and - optionally - managed resources. + /// + /// + /// true when called from Dispose; false when called from finalizer. + /// + protected virtual void Dispose(bool disposing) + { + Target.Dispose(); + } + + /// + /// Throws if this instance has been disposed. + /// + protected void ThrowIfDisposed() + { + if (_disposed) + throw new ObjectDisposedException(GetType().Name, "Object has been disposed."); + } + } +} \ No newline at end of file diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/PlatformJniTarget.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/PlatformJniTarget.cs new file mode 100644 index 0000000000000..725c112a0280d --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/PlatformJniTarget.cs @@ -0,0 +1,536 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Impl +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Diagnostics.CodeAnalysis; + using System.IO; + using System.Threading; + using System.Threading.Tasks; + using Apache.Ignite.Core.Binary; + using Apache.Ignite.Core.Impl.Binary; + using Apache.Ignite.Core.Impl.Binary.IO; + using Apache.Ignite.Core.Impl.Common; + using Apache.Ignite.Core.Impl.Memory; + using Apache.Ignite.Core.Impl.Unmanaged; + using Apache.Ignite.Core.Interop; + using BinaryReader = Apache.Ignite.Core.Impl.Binary.BinaryReader; + using BinaryWriter = Apache.Ignite.Core.Impl.Binary.BinaryWriter; + using UU = Apache.Ignite.Core.Impl.Unmanaged.UnmanagedUtils; + + /// + /// Base class for interop targets. + /// + internal class PlatformJniTarget : IPlatformTargetInternal + { + /** */ + private static readonly Dictionary IgniteFutureTypeMap + = new Dictionary + { + {typeof(bool), FutureType.Bool}, + {typeof(byte), FutureType.Byte}, + {typeof(char), FutureType.Char}, + {typeof(double), FutureType.Double}, + {typeof(float), FutureType.Float}, + {typeof(int), FutureType.Int}, + {typeof(long), FutureType.Long}, + {typeof(short), FutureType.Short} + }; + + /** Unmanaged target. */ + private readonly IUnmanagedTarget _target; + + /** Marshaller. */ + private readonly Marshaller _marsh; + + /// + /// Constructor. + /// + /// Target. + /// Marshaller. + public PlatformJniTarget(IUnmanagedTarget target, Marshaller marsh) + { + Debug.Assert(target != null); + Debug.Assert(marsh != null); + + _target = target; + _marsh = marsh; + } + + /// + /// Gets the target. + /// + public IUnmanagedTarget Target + { + get { return _target; } + } + + /** */ + public Marshaller Marshaller { get { return _marsh; } } + + /** */ + public long InStreamOutLong(int type, Action writeAction) + { + using (var stream = IgniteManager.Memory.Allocate().GetStream()) + { + writeAction(stream); + + return UU.TargetInStreamOutLong(_target, type, stream.SynchronizeOutput()); + } + } + + /** */ + public IPlatformTargetInternal InStreamOutObject(int type, Action writeAction) + { + using (var stream = IgniteManager.Memory.Allocate().GetStream()) + { + writeAction(stream); + + var target = UU.TargetInStreamOutObject(_target, type, stream.SynchronizeOutput()); + + return target == null ? null : new PlatformJniTarget(target, _marsh); + } + } + + /** */ + public IPlatformTargetInternal OutObjectInternal(int type) + { + return GetPlatformTarget(UU.TargetOutObject(_target, type)); + } + + /** */ + public T OutStream(int type, Func readAction) + { + using (var stream = IgniteManager.Memory.Allocate().GetStream()) + { + UU.TargetOutStream(_target, type, stream.MemoryPointer); + + stream.SynchronizeInput(); + + return readAction(stream); + } + } + + /** */ + public TR InStreamOutStream(int type, Action writeAction, + Func readAction) + { + using (var outStream = IgniteManager.Memory.Allocate().GetStream()) + using (var inStream = IgniteManager.Memory.Allocate().GetStream()) + { + writeAction(outStream); + + UU.TargetInStreamOutStream(_target, type, outStream.SynchronizeOutput(), inStream.MemoryPointer); + + inStream.SynchronizeInput(); + + return readAction(inStream); + } + } + + /** */ + public TR InStreamOutLong(int type, Action outAction, Func inAction, + Func readErrorAction) + { + Debug.Assert(readErrorAction != null); + + using (var stream = IgniteManager.Memory.Allocate().GetStream()) + { + outAction(stream); + + var res = UU.TargetInStreamOutLong(_target, type, stream.SynchronizeOutput()); + + if (res != PlatformTargetAdapter.Error && inAction == null) + return default(TR); // quick path for void operations + + stream.SynchronizeInput(); + + stream.Seek(0, SeekOrigin.Begin); + + if (res != PlatformTargetAdapter.Error) + { + return inAction != null ? inAction(stream, res) : default(TR); + } + + throw readErrorAction(stream); + } + } + + /** */ + public unsafe TR InObjectStreamOutObjectStream(int type, Action writeAction, + Func readAction, IPlatformTargetInternal arg) + { + PlatformMemoryStream outStream = null; + long outPtr = 0; + + PlatformMemoryStream inStream = null; + long inPtr = 0; + + try + { + if (writeAction != null) + { + outStream = IgniteManager.Memory.Allocate().GetStream(); + writeAction(outStream); + outPtr = outStream.SynchronizeOutput(); + } + + if (readAction != null) + { + inStream = IgniteManager.Memory.Allocate().GetStream(); + inPtr = inStream.MemoryPointer; + } + + var res = UU.TargetInObjectStreamOutObjectStream(_target, type, + ((PlatformJniTarget)arg).Target.Target, outPtr, inPtr); + + if (readAction == null) + return default(TR); + + inStream.SynchronizeInput(); + + var target = res == null ? null : new PlatformJniTarget(res, _marsh); + + return readAction(inStream, target); + + } + finally + { + try + { + if (inStream != null) + inStream.Dispose(); + + } + finally + { + if (outStream != null) + outStream.Dispose(); + } + } + } + + /// + /// Finish marshaling. + /// + /// Writer. + private void FinishMarshal(BinaryWriter writer) + { + _marsh.FinishMarshal(writer); + } + + /// + /// Creates a future and starts listening. + /// + /// Future result type + /// The listen action. + /// Keep binary flag, only applicable to object futures. False by default. + /// The function to read future result from stream. + /// Created future. + private Future GetFuture(Func listenAction, bool keepBinary = false, + Func convertFunc = null) + { + var futType = FutureType.Object; + + var type = typeof(T); + + if (type.IsPrimitive) + IgniteFutureTypeMap.TryGetValue(type, out futType); + + var fut = convertFunc == null && futType != FutureType.Object + ? new Future() + : new Future(new FutureConverter(_marsh, keepBinary, convertFunc)); + + var futHnd = _marsh.Ignite.HandleRegistry.Allocate(fut); + + IUnmanagedTarget futTarget; + + try + { + futTarget = listenAction(futHnd, (int)futType); + } + catch (Exception) + { + _marsh.Ignite.HandleRegistry.Release(futHnd); + + throw; + } + + fut.SetTarget(new Listenable(new PlatformJniTarget(futTarget, _marsh))); + + return fut; + } + + /// + /// Creates a future and starts listening. + /// + /// Future result type + /// The listen action. + /// Keep binary flag, only applicable to object futures. False by default. + /// The function to read future result from stream. + /// Created future. + private Future GetFuture(Action listenAction, bool keepBinary = false, + Func convertFunc = null) + { + var futType = FutureType.Object; + + var type = typeof(T); + + if (type.IsPrimitive) + IgniteFutureTypeMap.TryGetValue(type, out futType); + + var fut = convertFunc == null && futType != FutureType.Object + ? new Future() + : new Future(new FutureConverter(_marsh, keepBinary, convertFunc)); + + var futHnd = _marsh.Ignite.HandleRegistry.Allocate(fut); + + try + { + listenAction(futHnd, (int)futType); + } + catch (Exception) + { + _marsh.Ignite.HandleRegistry.Release(futHnd); + + throw; + } + + return fut; + } + + #region IPlatformTarget + + /** */ + public long InLongOutLong(int type, long val) + { + return UU.TargetInLongOutLong(_target, type, val); + } + + /** */ + public long InStreamOutLong(int type, Action writeAction) + { + using (var stream = IgniteManager.Memory.Allocate().GetStream()) + { + var writer = _marsh.StartMarshal(stream); + + writeAction(writer); + + FinishMarshal(writer); + + return UU.TargetInStreamOutLong(_target, type, stream.SynchronizeOutput()); + } + } + + /** */ + public T InStreamOutStream(int type, Action writeAction, + Func readAction) + { + using (var outStream = IgniteManager.Memory.Allocate().GetStream()) + using (var inStream = IgniteManager.Memory.Allocate().GetStream()) + { + var writer = _marsh.StartMarshal(outStream); + + writeAction(writer); + + FinishMarshal(writer); + + UU.TargetInStreamOutStream(_target, type, outStream.SynchronizeOutput(), inStream.MemoryPointer); + + inStream.SynchronizeInput(); + + return readAction(_marsh.StartUnmarshal(inStream)); + } + } + + /** */ + public IPlatformTarget InStreamOutObject(int type, Action writeAction) + { + using (var stream = IgniteManager.Memory.Allocate().GetStream()) + { + var writer = _marsh.StartMarshal(stream); + + writeAction(writer); + + FinishMarshal(writer); + + return GetPlatformTarget(UU.TargetInStreamOutObject(_target, type, stream.SynchronizeOutput())); + } + } + + /** */ + public unsafe T InObjectStreamOutObjectStream(int type, IPlatformTarget arg, + Action writeAction, Func readAction) + { + PlatformMemoryStream outStream = null; + long outPtr = 0; + + PlatformMemoryStream inStream = null; + long inPtr = 0; + + try + { + if (writeAction != null) + { + outStream = IgniteManager.Memory.Allocate().GetStream(); + var writer = _marsh.StartMarshal(outStream); + writeAction(writer); + FinishMarshal(writer); + outPtr = outStream.SynchronizeOutput(); + } + + if (readAction != null) + { + inStream = IgniteManager.Memory.Allocate().GetStream(); + inPtr = inStream.MemoryPointer; + } + + var res = UU.TargetInObjectStreamOutObjectStream(_target, type, GetTargetPtr(arg), outPtr, inPtr); + + if (readAction == null) + return default(T); + + inStream.SynchronizeInput(); + + return readAction(_marsh.StartUnmarshal(inStream), GetPlatformTarget(res)); + + } + finally + { + try + { + if (inStream != null) + inStream.Dispose(); + + } + finally + { + if (outStream != null) + outStream.Dispose(); + } + } + } + + /** */ + public T OutStream(int type, Func readAction) + { + using (var stream = IgniteManager.Memory.Allocate().GetStream()) + { + UU.TargetOutStream(_target, type, stream.MemoryPointer); + + stream.SynchronizeInput(); + + return readAction(_marsh.StartUnmarshal(stream)); + } + } + + /** */ + public IPlatformTarget OutObject(int type) + { + return OutObjectInternal(type); + } + + /** */ + public Task DoOutOpAsync(int type, Action writeAction = null, + Func readAction = null) + { + var convertFunc = readAction != null + ? r => readAction(r) + : (Func)null; + return GetFuture((futId, futType) => + { + using (var stream = IgniteManager.Memory.Allocate().GetStream()) + { + stream.WriteLong(futId); + stream.WriteInt(futType); + + if (writeAction != null) + { + var writer = _marsh.StartMarshal(stream); + + writeAction(writer); + + FinishMarshal(writer); + } + + UU.TargetInStreamAsync(_target, type, stream.SynchronizeOutput()); + } + }, false, convertFunc).Task; + } + + /** */ + public Task DoOutOpAsync(int type, Action writeAction, + Func readAction, CancellationToken cancellationToken) + { + var convertFunc = readAction != null + ? r => readAction(r) + : (Func) null; + + return GetFuture((futId, futType) => + { + using (var stream = IgniteManager.Memory.Allocate().GetStream()) + { + stream.WriteLong(futId); + stream.WriteInt(futType); + + if (writeAction != null) + { + var writer = _marsh.StartMarshal(stream); + + writeAction(writer); + + FinishMarshal(writer); + } + + return UU.TargetInStreamOutObjectAsync(_target, type, stream.SynchronizeOutput()); + } + }, false, convertFunc).GetTask(cancellationToken); + } + + /// + /// Gets the platform target. + /// + private IPlatformTargetInternal GetPlatformTarget(IUnmanagedTarget target) + { + return target == null ? null : new PlatformJniTarget(target, _marsh); + } + + /// + /// Gets the target pointer. + /// + private static unsafe void* GetTargetPtr(IPlatformTarget target) + { + return target == null ? null : ((PlatformJniTarget) target)._target.Target; + } + + #endregion + + /** */ + [SuppressMessage("Microsoft.Usage", "CA1816:CallGCSuppressFinalizeCorrectly", + Justification = "There is no finalizer.")] + public void Dispose() + { + if (_target != null) + { + _target.Dispose(); + } + } + } +} diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/PlatformTarget.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/PlatformTarget.cs deleted file mode 100644 index 474af0e37b961..0000000000000 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/PlatformTarget.cs +++ /dev/null @@ -1,1086 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -namespace Apache.Ignite.Core.Impl -{ - using System; - using System.Collections.Generic; - using System.Diagnostics; - using System.Diagnostics.CodeAnalysis; - using System.IO; - using System.Threading.Tasks; - using Apache.Ignite.Core.Binary; - using Apache.Ignite.Core.Impl.Binary; - using Apache.Ignite.Core.Impl.Binary.IO; - using Apache.Ignite.Core.Impl.Common; - using Apache.Ignite.Core.Impl.Memory; - using Apache.Ignite.Core.Impl.Unmanaged; - using Apache.Ignite.Core.Interop; - using BinaryReader = Apache.Ignite.Core.Impl.Binary.BinaryReader; - using BinaryWriter = Apache.Ignite.Core.Impl.Binary.BinaryWriter; - using UU = Apache.Ignite.Core.Impl.Unmanaged.UnmanagedUtils; - - /// - /// Base class for interop targets. - /// - [SuppressMessage("ReSharper", "LocalVariableHidesMember")] - internal class PlatformTarget : IPlatformTarget - { - /** */ - protected const int False = 0; - - /** */ - protected const int True = 1; - - /** */ - protected const int Error = -1; - - /** */ - public const int OpNone = -2; - - /** */ - private static readonly Dictionary IgniteFutureTypeMap - = new Dictionary - { - {typeof(bool), FutureType.Bool}, - {typeof(byte), FutureType.Byte}, - {typeof(char), FutureType.Char}, - {typeof(double), FutureType.Double}, - {typeof(float), FutureType.Float}, - {typeof(int), FutureType.Int}, - {typeof(long), FutureType.Long}, - {typeof(short), FutureType.Short} - }; - - /** Unmanaged target. */ - private readonly IUnmanagedTarget _target; - - /** Marshaller. */ - private readonly Marshaller _marsh; - - /// - /// Constructor. - /// - /// Target. - /// Marshaller. - public PlatformTarget(IUnmanagedTarget target, Marshaller marsh) - { - Debug.Assert(target != null); - Debug.Assert(marsh != null); - - _target = target; - _marsh = marsh; - } - - /// - /// Unmanaged target. - /// - internal IUnmanagedTarget Target - { - get { return _target; } - } - - /// - /// Marshaller. - /// - internal Marshaller Marshaller - { - get { return _marsh; } - } - - #region Static Helpers - - /// - /// Write collection. - /// - /// Writer. - /// Values. - /// The same writer for chaining. - protected static BinaryWriter WriteCollection(BinaryWriter writer, ICollection vals) - { - return WriteCollection(writer, vals, null); - } - - /// - /// Write nullable collection. - /// - /// Writer. - /// Values. - /// The same writer for chaining. - protected static BinaryWriter WriteNullableCollection(BinaryWriter writer, ICollection vals) - { - return WriteNullable(writer, vals, WriteCollection); - } - - /// - /// Write collection. - /// - /// Writer. - /// Values. - /// A transform function to apply to each element. - /// The same writer for chaining. - protected static BinaryWriter WriteCollection(BinaryWriter writer, - ICollection vals, Func selector) - { - writer.WriteInt(vals.Count); - - if (selector == null) - { - foreach (var val in vals) - writer.Write(val); - } - else - { - foreach (var val in vals) - writer.Write(selector(val)); - } - - return writer; - } - - /// - /// Write enumerable. - /// - /// Writer. - /// Values. - /// The same writer for chaining. - protected static BinaryWriter WriteEnumerable(BinaryWriter writer, IEnumerable vals) - { - return WriteEnumerable(writer, vals, null); - } - - /// - /// Write enumerable. - /// - /// Writer. - /// Values. - /// A transform function to apply to each element. - /// The same writer for chaining. - protected static BinaryWriter WriteEnumerable(BinaryWriter writer, - IEnumerable vals, Func selector) - { - var col = vals as ICollection; - - if (col != null) - return WriteCollection(writer, col, selector); - - var stream = writer.Stream; - - var pos = stream.Position; - - stream.Seek(4, SeekOrigin.Current); - - var size = 0; - - if (selector == null) - { - foreach (var val in vals) - { - writer.Write(val); - - size++; - } - } - else - { - foreach (var val in vals) - { - writer.Write(selector(val)); - - size++; - } - } - - stream.WriteInt(pos, size); - - return writer; - } - - /// - /// Write dictionary. - /// - /// Writer. - /// Values. - protected static void WriteDictionary(BinaryWriter writer, IEnumerable> vals) - { - var pos = writer.Stream.Position; - writer.WriteInt(0); // Reserve count. - - int cnt = 0; - - foreach (var pair in vals) - { - writer.Write(pair.Key); - writer.Write(pair.Value); - - cnt++; - } - - writer.Stream.WriteInt(pos, cnt); - } - - /// - /// Write a nullable item. - /// - /// Writer. - /// Item. - /// Write action to perform on item when it is not null. - /// The same writer for chaining. - private static BinaryWriter WriteNullable(BinaryWriter writer, T item, - Func writeItem) where T : class - { - if (item == null) - { - writer.WriteBoolean(false); - - return writer; - } - - writer.WriteBoolean(true); - - return writeItem(writer, item); - } - - #endregion - - #region OUT operations - - /// - /// Perform out operation. - /// - /// Operation type. - /// Action to be performed on the stream. - /// - protected long DoOutOp(int type, Action action) - { - using (var stream = IgniteManager.Memory.Allocate().GetStream()) - { - action(stream); - - return UU.TargetInStreamOutLong(_target, type, stream.SynchronizeOutput()); - } - } - - /// - /// Perform out operation. - /// - /// Operation type. - /// Action to be performed on the stream. - /// - protected long DoOutOp(int type, Action action) - { - using (var stream = IgniteManager.Memory.Allocate().GetStream()) - { - var writer = _marsh.StartMarshal(stream); - - action(writer); - - FinishMarshal(writer); - - return UU.TargetInStreamOutLong(_target, type, stream.SynchronizeOutput()); - } - } - - /// - /// Perform out operation. - /// - /// Operation type. - /// Action to be performed on the stream. - /// Resulting object. - protected IUnmanagedTarget DoOutOpObject(int type, Action action) - { - using (var stream = IgniteManager.Memory.Allocate().GetStream()) - { - var writer = _marsh.StartMarshal(stream); - - action(writer); - - FinishMarshal(writer); - - return UU.TargetInStreamOutObject(_target, type, stream.SynchronizeOutput()); - } - } - - /// - /// Perform out operation. - /// - /// Operation type. - /// Action to be performed on the stream. - /// Resulting object. - protected IUnmanagedTarget DoOutOpObject(int type, Action action) - { - using (var stream = IgniteManager.Memory.Allocate().GetStream()) - { - action(stream); - - return UU.TargetInStreamOutObject(_target, type, stream.SynchronizeOutput()); - } - } - - /// - /// Perform out operation. - /// - /// Operation type. - /// Resulting object. - protected IUnmanagedTarget DoOutOpObject(int type) - { - return UU.TargetOutObject(_target, type); - } - - /// - /// Perform simple output operation accepting single argument. - /// - /// Operation type. - /// Value. - /// Result. - protected long DoOutOp(int type, T1 val1) - { - return DoOutOp(type, writer => - { - writer.Write(val1); - }); - } - - /// - /// Perform simple output operation accepting two arguments. - /// - /// Operation type. - /// Value 1. - /// Value 2. - /// Result. - protected long DoOutOp(int type, T1 val1, T2 val2) - { - return DoOutOp(type, writer => - { - writer.Write(val1); - writer.Write(val2); - }); - } - - /// - /// Perform simple output operation accepting three arguments. - /// - /// Operation type. - /// Value 1. - /// Value 2. - /// Value 3. - /// Result. - protected long DoOutOp(int type, T1 val1, T2 val2, T3 val3) - { - return DoOutOp(type, writer => - { - writer.Write(val1); - writer.Write(val2); - writer.Write(val3); - }); - } - - #endregion - - #region IN operations - - /// - /// Perform in operation. - /// - /// Type. - /// Action. - protected void DoInOp(int type, Action action) - { - using (var stream = IgniteManager.Memory.Allocate().GetStream()) - { - UU.TargetOutStream(_target, type, stream.MemoryPointer); - - stream.SynchronizeInput(); - - action(stream); - } - } - - /// - /// Perform in operation. - /// - /// Type. - /// Action. - /// Result. - protected T DoInOp(int type, Func action) - { - using (var stream = IgniteManager.Memory.Allocate().GetStream()) - { - UU.TargetOutStream(_target, type, stream.MemoryPointer); - - stream.SynchronizeInput(); - - return action(stream); - } - } - - /// - /// Perform simple in operation returning immediate result. - /// - /// Type. - /// Result. - protected T DoInOp(int type) - { - using (var stream = IgniteManager.Memory.Allocate().GetStream()) - { - UU.TargetOutStream(_target, type, stream.MemoryPointer); - - stream.SynchronizeInput(); - - return Unmarshal(stream); - } - } - - #endregion - - #region OUT-IN operations - - /// - /// Perform out-in operation. - /// - /// Operation type. - /// Out action. - /// In action. - protected void DoOutInOp(int type, Action outAction, Action inAction) - { - using (PlatformMemoryStream outStream = IgniteManager.Memory.Allocate().GetStream()) - { - using (PlatformMemoryStream inStream = IgniteManager.Memory.Allocate().GetStream()) - { - BinaryWriter writer = _marsh.StartMarshal(outStream); - - outAction(writer); - - FinishMarshal(writer); - - UU.TargetInStreamOutStream(_target, type, outStream.SynchronizeOutput(), inStream.MemoryPointer); - - inStream.SynchronizeInput(); - - inAction(inStream); - } - } - } - - /// - /// Perform out-in operation. - /// - /// Operation type. - /// Out action. - /// In action. - /// Result. - protected TR DoOutInOp(int type, Action outAction, Func inAction) - { - using (PlatformMemoryStream outStream = IgniteManager.Memory.Allocate().GetStream()) - { - using (PlatformMemoryStream inStream = IgniteManager.Memory.Allocate().GetStream()) - { - BinaryWriter writer = _marsh.StartMarshal(outStream); - - outAction(writer); - - FinishMarshal(writer); - - UU.TargetInStreamOutStream(_target, type, outStream.SynchronizeOutput(), inStream.MemoryPointer); - - inStream.SynchronizeInput(); - - return inAction(inStream); - } - } - } - - /// - /// Perform out-in operation with a single stream. - /// - /// The type of the r. - /// Operation type. - /// Out action. - /// In action. - /// The action to read an error. - /// - /// Result. - /// - protected TR DoOutInOpX(int type, Action outAction, Func inAction, - Func inErrorAction) - { - Debug.Assert(inErrorAction != null); - - using (var stream = IgniteManager.Memory.Allocate().GetStream()) - { - var writer = _marsh.StartMarshal(stream); - - outAction(writer); - - FinishMarshal(writer); - - var res = UU.TargetInStreamOutLong(_target, type, stream.SynchronizeOutput()); - - if (res != Error && inAction == null) - return default(TR); // quick path for void operations - - stream.SynchronizeInput(); - - stream.Seek(0, SeekOrigin.Begin); - - if (res != Error) - return inAction != null ? inAction(stream, res) : default(TR); - - throw inErrorAction(stream); - } - } - - /// - /// Perform out-in operation with a single stream. - /// - /// Operation type. - /// Out action. - /// The action to read an error. - /// - /// Result. - /// - protected bool DoOutInOpX(int type, Action outAction, - Func inErrorAction) - { - Debug.Assert(inErrorAction != null); - - using (var stream = IgniteManager.Memory.Allocate().GetStream()) - { - var writer = _marsh.StartMarshal(stream); - - outAction(writer); - - FinishMarshal(writer); - - var res = UU.TargetInStreamOutLong(_target, type, stream.SynchronizeOutput()); - - if (res != Error) - return res == True; - - stream.SynchronizeInput(); - - stream.Seek(0, SeekOrigin.Begin); - - throw inErrorAction(stream); - } - } - - /// - /// Perform out-in operation. - /// - /// Operation type. - /// Out action. - /// In action. - /// Argument. - /// Result. - protected unsafe TR DoOutInOp(int type, Action outAction, - Func inAction, void* arg) - { - PlatformMemoryStream outStream = null; - long outPtr = 0; - - PlatformMemoryStream inStream = null; - long inPtr = 0; - - try - { - if (outAction != null) - { - outStream = IgniteManager.Memory.Allocate().GetStream(); - var writer = _marsh.StartMarshal(outStream); - outAction(writer); - FinishMarshal(writer); - outPtr = outStream.SynchronizeOutput(); - } - - if (inAction != null) - { - inStream = IgniteManager.Memory.Allocate().GetStream(); - inPtr = inStream.MemoryPointer; - } - - var res = UU.TargetInObjectStreamOutObjectStream(_target, type, arg, outPtr, inPtr); - - if (inAction == null) - return default(TR); - - inStream.SynchronizeInput(); - - return inAction(inStream, res); - - } - finally - { - try - { - if (inStream != null) - inStream.Dispose(); - - } - finally - { - if (outStream != null) - outStream.Dispose(); - } - } - } - - /// - /// Perform out-in operation. - /// - /// Operation type. - /// Out action. - /// Result. - protected TR DoOutInOp(int type, Action outAction) - { - using (PlatformMemoryStream outStream = IgniteManager.Memory.Allocate().GetStream()) - { - using (PlatformMemoryStream inStream = IgniteManager.Memory.Allocate().GetStream()) - { - BinaryWriter writer = _marsh.StartMarshal(outStream); - - outAction(writer); - - FinishMarshal(writer); - - UU.TargetInStreamOutStream(_target, type, outStream.SynchronizeOutput(), inStream.MemoryPointer); - - inStream.SynchronizeInput(); - - return Unmarshal(inStream); - } - } - } - - /// - /// Perform simple out-in operation accepting single argument. - /// - /// Operation type. - /// Value. - /// Result. - protected TR DoOutInOp(int type, T1 val) - { - using (PlatformMemoryStream outStream = IgniteManager.Memory.Allocate().GetStream()) - { - using (PlatformMemoryStream inStream = IgniteManager.Memory.Allocate().GetStream()) - { - BinaryWriter writer = _marsh.StartMarshal(outStream); - - writer.WriteObject(val); - - FinishMarshal(writer); - - UU.TargetInStreamOutStream(_target, type, outStream.SynchronizeOutput(), inStream.MemoryPointer); - - inStream.SynchronizeInput(); - - return Unmarshal(inStream); - } - } - } - - /// - /// Perform simple out-in operation accepting two arguments. - /// - /// Operation type. - /// Value. - /// Value. - /// Result. - protected TR DoOutInOp(int type, T1 val1, T2 val2) - { - using (PlatformMemoryStream outStream = IgniteManager.Memory.Allocate().GetStream()) - { - using (PlatformMemoryStream inStream = IgniteManager.Memory.Allocate().GetStream()) - { - BinaryWriter writer = _marsh.StartMarshal(outStream); - - writer.WriteObject(val1); - writer.WriteObject(val2); - - FinishMarshal(writer); - - UU.TargetInStreamOutStream(_target, type, outStream.SynchronizeOutput(), inStream.MemoryPointer); - - inStream.SynchronizeInput(); - - return Unmarshal(inStream); - } - } - } - - /// - /// Perform simple out-in operation accepting two arguments. - /// - /// Operation type. - /// Value. - /// Result. - protected long DoOutInOp(int type, long val = 0) - { - return UU.TargetInLongOutLong(_target, type, val); - } - - #endregion - - #region Async operations - - /// - /// Performs async operation. - /// - /// The type code. - /// The write action. - /// Task for async operation - protected Task DoOutOpAsync(int type, Action writeAction = null) - { - return DoOutOpAsync(type, writeAction); - } - - /// - /// Performs async operation. - /// - /// Type of the result. - /// The type code. - /// The write action. - /// Keep binary flag, only applicable to object futures. False by default. - /// The function to read future result from stream. - /// Task for async operation - protected Task DoOutOpAsync(int type, Action writeAction = null, bool keepBinary = false, - Func convertFunc = null) - { - return GetFuture((futId, futType) => DoOutOp(type, w => - { - if (writeAction != null) - writeAction(w); - w.WriteLong(futId); - w.WriteInt(futType); - }), keepBinary, convertFunc).Task; - } - - /// - /// Performs async operation. - /// - /// Type of the result. - /// The type code. - /// The write action. - /// Future for async operation - protected Future DoOutOpObjectAsync(int type, Action writeAction) - { - return GetFuture((futId, futType) => DoOutOpObject(type, w => - { - writeAction(w); - w.WriteLong(futId); - w.WriteInt(futType); - })); - } - - /// - /// Performs async operation. - /// - /// Type of the result. - /// The type of the first arg. - /// The type code. - /// First arg. - /// - /// Task for async operation - /// - protected Task DoOutOpAsync(int type, T1 val1) - { - return GetFuture((futId, futType) => DoOutOp(type, w => - { - w.WriteObject(val1); - w.WriteLong(futId); - w.WriteInt(futType); - })).Task; - } - - /// - /// Performs async operation. - /// - /// Type of the result. - /// The type of the first arg. - /// The type of the second arg. - /// The type code. - /// First arg. - /// Second arg. - /// - /// Task for async operation - /// - protected Task DoOutOpAsync(int type, T1 val1, T2 val2) - { - return GetFuture((futId, futType) => DoOutOp(type, w => - { - w.WriteObject(val1); - w.WriteObject(val2); - w.WriteLong(futId); - w.WriteInt(futType); - })).Task; - } - - #endregion - - #region Miscelanneous - - /// - /// Finish marshaling. - /// - /// Writer. - internal void FinishMarshal(BinaryWriter writer) - { - _marsh.FinishMarshal(writer); - } - - /// - /// Unmarshal object using the given stream. - /// - /// Stream. - /// Unmarshalled object. - protected virtual T Unmarshal(IBinaryStream stream) - { - return _marsh.Unmarshal(stream); - } - - /// - /// Creates a future and starts listening. - /// - /// Future result type - /// The listen action. - /// Keep binary flag, only applicable to object futures. False by default. - /// The function to read future result from stream. - /// Created future. - private Future GetFuture(Func listenAction, bool keepBinary = false, - Func convertFunc = null) - { - var futType = FutureType.Object; - - var type = typeof(T); - - if (type.IsPrimitive) - IgniteFutureTypeMap.TryGetValue(type, out futType); - - var fut = convertFunc == null && futType != FutureType.Object - ? new Future() - : new Future(new FutureConverter(_marsh, keepBinary, convertFunc)); - - var futHnd = _marsh.Ignite.HandleRegistry.Allocate(fut); - - IUnmanagedTarget futTarget; - - try - { - futTarget = listenAction(futHnd, (int)futType); - } - catch (Exception) - { - _marsh.Ignite.HandleRegistry.Release(futHnd); - - throw; - } - - fut.SetTarget(new Listenable(futTarget, _marsh)); - - return fut; - } - - /// - /// Creates a future and starts listening. - /// - /// Future result type - /// The listen action. - /// Keep binary flag, only applicable to object futures. False by default. - /// The function to read future result from stream. - /// Created future. - protected Future GetFuture(Action listenAction, bool keepBinary = false, - Func convertFunc = null) - { - var futType = FutureType.Object; - - var type = typeof(T); - - if (type.IsPrimitive) - IgniteFutureTypeMap.TryGetValue(type, out futType); - - var fut = convertFunc == null && futType != FutureType.Object - ? new Future() - : new Future(new FutureConverter(_marsh, keepBinary, convertFunc)); - - var futHnd = _marsh.Ignite.HandleRegistry.Allocate(fut); - - try - { - listenAction(futHnd, (int)futType); - } - catch (Exception) - { - _marsh.Ignite.HandleRegistry.Release(futHnd); - - throw; - } - - return fut; - } - - #endregion - - #region IPlatformTarget - - /** */ - public long InLongOutLong(int type, long val) - { - return DoOutInOp(type, val); - } - - /** */ - public long InStreamOutLong(int type, Action writeAction) - { - return DoOutOp(type, writer => writeAction(writer)); - } - - /** */ - public T InStreamOutStream(int type, Action writeAction, - Func readAction) - { - return DoOutInOp(type, writeAction, stream => readAction(Marshaller.StartUnmarshal(stream))); - } - - /** */ - public IPlatformTarget InStreamOutObject(int type, Action writeAction) - { - return GetPlatformTarget(DoOutOpObject(type, writeAction)); - } - - /** */ - public unsafe T InObjectStreamOutObjectStream(int type, IPlatformTarget arg, Action writeAction, - Func readAction) - { - return DoOutInOp(type, writeAction, (stream, obj) => readAction(Marshaller.StartUnmarshal(stream), - GetPlatformTarget(obj)), GetTargetPtr(arg)); - } - - /** */ - public T OutStream(int type, Func readAction) - { - return DoInOp(type, stream => readAction(Marshaller.StartUnmarshal(stream))); - } - - /** */ - public IPlatformTarget OutObject(int type) - { - return GetPlatformTarget(DoOutOpObject(type)); - } - - /** */ - public Task DoOutOpAsync(int type, Action writeAction = null, - Func readAction = null) - { - var convertFunc = readAction != null - ? r => readAction(r) - : (Func) null; - - return GetFuture((futId, futType) => - { - using (var stream = IgniteManager.Memory.Allocate().GetStream()) - { - stream.WriteLong(futId); - stream.WriteInt(futType); - - if (writeAction != null) - { - var writer = _marsh.StartMarshal(stream); - - writeAction(writer); - - FinishMarshal(writer); - } - - UU.TargetInStreamAsync(_target, type, stream.SynchronizeOutput()); - } - }, false, convertFunc).Task; - } - - /// - /// Gets the platform target. - /// - private IPlatformTarget GetPlatformTarget(IUnmanagedTarget target) - { - return target == null ? null : new PlatformTarget(target, Marshaller); - } - - /// - /// Gets the target pointer. - /// - private static unsafe void* GetTargetPtr(IPlatformTarget target) - { - return target == null ? null : ((PlatformTarget) target).Target.Target; - } - - #endregion - } - - /// - /// PlatformTarget with IDisposable pattern. - /// - internal abstract class PlatformDisposableTarget : PlatformTarget, IDisposable - { - /** Disposed flag. */ - private volatile bool _disposed; - - /// - /// Constructor. - /// - /// Target. - /// Marshaller. - protected PlatformDisposableTarget(IUnmanagedTarget target, Marshaller marsh) : base(target, marsh) - { - // No-op. - } - - /** */ - public void Dispose() - { - lock (this) - { - if (_disposed) - return; - - Dispose(true); - - GC.SuppressFinalize(this); - - _disposed = true; - } - } - - /// - /// Releases unmanaged and - optionally - managed resources. - /// - /// - /// true when called from Dispose; false when called from finalizer. - /// - protected virtual void Dispose(bool disposing) - { - Target.Dispose(); - } - - /// - /// Throws if this instance has been disposed. - /// - protected void ThrowIfDisposed() - { - if (_disposed) - throw new ObjectDisposedException(GetType().Name, "Object has been disposed."); - } - } -} diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/PlatformTargetAdapter.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/PlatformTargetAdapter.cs new file mode 100644 index 0000000000000..64b5f29673a61 --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/PlatformTargetAdapter.cs @@ -0,0 +1,534 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Impl +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Diagnostics.CodeAnalysis; + using System.Threading.Tasks; + using Apache.Ignite.Core.Binary; + using Apache.Ignite.Core.Impl.Binary; + using Apache.Ignite.Core.Impl.Binary.IO; + using Apache.Ignite.Core.Impl.Common; + using BinaryReader = Apache.Ignite.Core.Impl.Binary.BinaryReader; + using BinaryWriter = Apache.Ignite.Core.Impl.Binary.BinaryWriter; + + /// + /// Base class for interop targets, provides additional functionality over . + /// + [SuppressMessage("ReSharper", "LocalVariableHidesMember")] + internal class PlatformTargetAdapter + { + /** */ + internal const int False = 0; + + /** */ + internal const int True = 1; + + /** */ + internal const int Error = -1; + + /** */ + private static readonly Dictionary IgniteFutureTypeMap + = new Dictionary + { + {typeof(bool), FutureType.Bool}, + {typeof(byte), FutureType.Byte}, + {typeof(char), FutureType.Char}, + {typeof(double), FutureType.Double}, + {typeof(float), FutureType.Float}, + {typeof(int), FutureType.Int}, + {typeof(long), FutureType.Long}, + {typeof(short), FutureType.Short} + }; + + /** Unmanaged target. */ + private readonly IPlatformTargetInternal _target; + + /** Marshaller. */ + private readonly Marshaller _marsh; + + /// + /// Constructor. + /// + /// Target. + protected PlatformTargetAdapter(IPlatformTargetInternal target) + { + Debug.Assert(target != null); + + _target = target; + _marsh = target.Marshaller; + } + + /// + /// Unmanaged target. + /// + internal IPlatformTargetInternal Target + { + get { return _target; } + } + + /// + /// Marshaller. + /// + internal Marshaller Marshaller + { + get { return _marsh; } + } + + #region OUT operations + + /// + /// Perform out operation. + /// + /// Operation type. + /// Action to be performed on the stream. + /// + protected long DoOutOp(int type, Action action) + { + return _target.InStreamOutLong(type, action); + } + + /// + /// Perform out operation. + /// + /// Operation type. + /// Action to be performed on the stream. + /// + protected long DoOutOp(int type, Action action) + { + return DoOutOp(type, stream => WriteToStream(action, stream, _marsh)); + } + + /// + /// Perform out operation. + /// + /// Operation type. + /// Action to be performed on the stream. + /// Resulting object. + protected IPlatformTargetInternal DoOutOpObject(int type, Action action) + { + return _target.InStreamOutObject(type, stream => WriteToStream(action, stream, _marsh)); + } + + /// + /// Perform out operation. + /// + /// Operation type. + /// Action to be performed on the stream. + /// Resulting object. + protected IPlatformTargetInternal DoOutOpObject(int type, Action action) + { + return _target.InStreamOutObject(type, action); + } + + /// + /// Perform out operation. + /// + /// Operation type. + /// Resulting object. + protected IPlatformTargetInternal DoOutOpObject(int type) + { + return _target.OutObjectInternal(type); + } + + /// + /// Perform simple output operation accepting single argument. + /// + /// Operation type. + /// Value. + /// Result. + protected long DoOutOp(int type, T1 val1) + { + return DoOutOp(type, writer => + { + writer.Write(val1); + }); + } + + /// + /// Perform simple output operation accepting two arguments. + /// + /// Operation type. + /// Value 1. + /// Value 2. + /// Result. + protected long DoOutOp(int type, T1 val1, T2 val2) + { + return DoOutOp(type, writer => + { + writer.Write(val1); + writer.Write(val2); + }); + } + + #endregion + + #region IN operations + + /// + /// Perform in operation. + /// + /// Type. + /// Action. + /// Result. + protected T DoInOp(int type, Func action) + { + return _target.OutStream(type, action); + } + + /// + /// Perform simple in operation returning immediate result. + /// + /// Type. + /// Result. + protected T DoInOp(int type) + { + return _target.OutStream(type, s => Unmarshal(s)); + } + + #endregion + + #region OUT-IN operations + + /// + /// Perform out-in operation. + /// + /// Operation type. + /// Out action. + /// In action. + /// Result. + protected TR DoOutInOp(int type, Action outAction, Func inAction) + { + return _target.InStreamOutStream(type, stream => WriteToStream(outAction, stream, _marsh), inAction); + } + + /// + /// Perform out-in operation with a single stream. + /// + /// The type of the r. + /// Operation type. + /// Out action. + /// In action. + /// The action to read an error. + /// + /// Result. + /// + protected TR DoOutInOpX(int type, Action outAction, Func inAction, + Func inErrorAction) + { + return _target.InStreamOutLong(type, stream => WriteToStream(outAction, stream, _marsh), + inAction, inErrorAction); + } + + /// + /// Perform out-in operation with a single stream. + /// + /// Operation type. + /// Out action. + /// The action to read an error. + /// + /// Result. + /// + protected bool DoOutInOpX(int type, Action outAction, + Func inErrorAction) + { + return _target.InStreamOutLong(type, stream => WriteToStream(outAction, stream, _marsh), + (stream, res) => res == True, inErrorAction); + } + + /// + /// Perform out-in operation. + /// + /// Operation type. + /// Out action. + /// In action. + /// Argument. + /// Result. + protected TR DoOutInOp(int type, Action outAction, + Func inAction, IPlatformTargetInternal arg) + { + return _target.InObjectStreamOutObjectStream(type, stream => WriteToStream(outAction, stream, _marsh), + inAction, arg); + } + + /// + /// Perform out-in operation. + /// + /// Operation type. + /// Out action. + /// Result. + protected TR DoOutInOp(int type, Action outAction) + { + return _target.InStreamOutStream(type, stream => WriteToStream(outAction, stream, _marsh), + stream => Unmarshal(stream)); + } + + /// + /// Perform simple out-in operation accepting single argument. + /// + /// Operation type. + /// Value. + /// Result. + protected TR DoOutInOp(int type, T1 val) + { + return _target.InStreamOutStream(type, stream => WriteToStream(val, stream, _marsh), + stream => Unmarshal(stream)); + } + + /// + /// Perform simple out-in operation accepting two arguments. + /// + /// Operation type. + /// Value. + /// Result. + protected long DoOutInOp(int type, long val = 0) + { + return _target.InLongOutLong(type, val); + } + + #endregion + + #region Async operations + + /// + /// Performs async operation. + /// + /// The type code. + /// The write action. + /// Task for async operation + protected Task DoOutOpAsync(int type, Action writeAction = null) + { + return DoOutOpAsync(type, writeAction); + } + + /// + /// Performs async operation. + /// + /// Type of the result. + /// The type code. + /// The write action. + /// Keep binary flag, only applicable to object futures. False by default. + /// The function to read future result from stream. + /// Task for async operation + protected Task DoOutOpAsync(int type, Action writeAction = null, bool keepBinary = false, + Func convertFunc = null) + { + return GetFuture((futId, futType) => DoOutOp(type, w => + { + if (writeAction != null) + { + writeAction(w); + } + w.WriteLong(futId); + w.WriteInt(futType); + }), keepBinary, convertFunc).Task; + } + + /// + /// Performs async operation. + /// + /// Type of the result. + /// The type code. + /// The write action. + /// Future for async operation + protected Future DoOutOpObjectAsync(int type, Action writeAction) + { + return GetFuture((futId, futType) => DoOutOpObject(type, w => + { + writeAction(w); + w.WriteLong(futId); + w.WriteInt(futType); + })); + } + + /// + /// Performs async operation. + /// + /// Type of the result. + /// The type of the first arg. + /// The type code. + /// First arg. + /// + /// Task for async operation + /// + protected Task DoOutOpAsync(int type, T1 val1) + { + return GetFuture((futId, futType) => DoOutOp(type, w => + { + w.WriteObject(val1); + w.WriteLong(futId); + w.WriteInt(futType); + })).Task; + } + + /// + /// Performs async operation. + /// + /// Type of the result. + /// The type of the first arg. + /// The type of the second arg. + /// The type code. + /// First arg. + /// Second arg. + /// + /// Task for async operation + /// + protected Task DoOutOpAsync(int type, T1 val1, T2 val2) + { + return GetFuture((futId, futType) => DoOutOp(type, w => + { + w.WriteObject(val1); + w.WriteObject(val2); + w.WriteLong(futId); + w.WriteInt(futType); + })).Task; + } + + #endregion + + #region Miscelanneous + + /// + /// Finish marshaling. + /// + /// Writer. + internal void FinishMarshal(BinaryWriter writer) + { + _marsh.FinishMarshal(writer); + } + + /// + /// Unmarshal object using the given stream. + /// + /// Stream. + /// Unmarshalled object. + protected virtual T Unmarshal(IBinaryStream stream) + { + return _marsh.Unmarshal(stream); + } + + /// + /// Creates a future and starts listening. + /// + /// Future result type + /// The listen action. + /// Keep binary flag, only applicable to object futures. False by default. + /// The function to read future result from stream. + /// Created future. + private Future GetFuture(Func listenAction, bool keepBinary = false, + Func convertFunc = null) + { + var futType = FutureType.Object; + + var type = typeof(T); + + if (type.IsPrimitive) + IgniteFutureTypeMap.TryGetValue(type, out futType); + + var fut = convertFunc == null && futType != FutureType.Object + ? new Future() + : new Future(new FutureConverter(_marsh, keepBinary, convertFunc)); + + var futHnd = _marsh.Ignite.HandleRegistry.Allocate(fut); + + IPlatformTargetInternal futTarget; + + try + { + futTarget = listenAction(futHnd, (int)futType); + } + catch (Exception) + { + _marsh.Ignite.HandleRegistry.Release(futHnd); + + throw; + } + + fut.SetTarget(new Listenable(futTarget)); + + return fut; + } + + /// + /// Creates a future and starts listening. + /// + /// Future result type + /// The listen action. + /// Keep binary flag, only applicable to object futures. False by default. + /// The function to read future result from stream. + /// Created future. + private Future GetFuture(Action listenAction, bool keepBinary = false, + Func convertFunc = null) + { + var futType = FutureType.Object; + + var type = typeof(T); + + if (type.IsPrimitive) + IgniteFutureTypeMap.TryGetValue(type, out futType); + + var fut = convertFunc == null && futType != FutureType.Object + ? new Future() + : new Future(new FutureConverter(_marsh, keepBinary, convertFunc)); + + var futHnd = _marsh.Ignite.HandleRegistry.Allocate(fut); + + try + { + listenAction(futHnd, (int)futType); + } + catch (Exception) + { + _marsh.Ignite.HandleRegistry.Release(futHnd); + + throw; + } + + return fut; + } + + /// + /// Writes to stream. + /// + private static void WriteToStream(Action action, IBinaryStream stream, Marshaller marsh) + { + var writer = marsh.StartMarshal(stream); + + action(writer); + + marsh.FinishMarshal(writer); + } + + /// + /// Writes to stream. + /// + private static void WriteToStream(T obj, IBinaryStream stream, Marshaller marsh) + { + var writer = marsh.StartMarshal(stream); + + writer.WriteObject(obj); + + marsh.FinishMarshal(writer); + } + + + #endregion + } +} diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Services/Services.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Services/Services.cs index 88d2a76f4dbf9..93611f7711655 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Services/Services.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Services/Services.cs @@ -24,16 +24,13 @@ namespace Apache.Ignite.Core.Impl.Services using System.Threading.Tasks; using Apache.Ignite.Core.Binary; using Apache.Ignite.Core.Cluster; - using Apache.Ignite.Core.Impl.Binary; using Apache.Ignite.Core.Impl.Common; - using Apache.Ignite.Core.Impl.Unmanaged; using Apache.Ignite.Core.Services; - using UU = Apache.Ignite.Core.Impl.Unmanaged.UnmanagedUtils; /// /// Services implementation. /// - internal sealed class Services : PlatformTarget, IServices + internal sealed class Services : PlatformTargetAdapter, IServices { /** */ private const int OpDeploy = 1; @@ -87,13 +84,12 @@ internal sealed class Services : PlatformTarget, IServices /// Initializes a new instance of the class. /// /// Target. - /// Marshaller. /// Cluster group. /// Invoker binary flag. /// Server binary flag. - public Services(IUnmanagedTarget target, Marshaller marsh, IClusterGroup clusterGroup, + public Services(IPlatformTargetInternal target, IClusterGroup clusterGroup, bool keepBinary, bool srvKeepBinary) - : base(target, marsh) + : base(target) { Debug.Assert(clusterGroup != null); @@ -108,7 +104,7 @@ public IServices WithKeepBinary() if (_keepBinary) return this; - return new Services(Target, Marshaller, _clusterGroup, true, _srvKeepBinary); + return new Services(Target, _clusterGroup, true, _srvKeepBinary); } /** */ @@ -117,7 +113,7 @@ public IServices WithServerKeepBinary() if (_srvKeepBinary) return this; - return new Services(DoOutOpObject(OpWithServerKeepBinary), Marshaller, _clusterGroup, _keepBinary, true); + return new Services(DoOutOpObject(OpWithServerKeepBinary), _clusterGroup, _keepBinary, true); } /** */ @@ -372,12 +368,13 @@ public ICollection GetServices(string name) /// /// Invocation result. /// - private unsafe object InvokeProxyMethod(IUnmanagedTarget proxy, MethodBase method, object[] args, + private object InvokeProxyMethod(IPlatformTargetInternal proxy, MethodBase method, object[] args, Platform platform) { return DoOutInOp(OpInvokeMethod, writer => ServiceProxySerializer.WriteProxyMethod(writer, method, args, platform), - (stream, res) => ServiceProxySerializer.ReadInvocationResult(stream, Marshaller, _keepBinary), proxy.Target); + (stream, res) => ServiceProxySerializer.ReadInvocationResult(stream, Marshaller, _keepBinary), + proxy); } /// diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Transactions/TransactionsImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Transactions/TransactionsImpl.cs index 4ddbc6ddf2018..4dd7f9f7de5e8 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Transactions/TransactionsImpl.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Transactions/TransactionsImpl.cs @@ -22,13 +22,12 @@ namespace Apache.Ignite.Core.Impl.Transactions using System.Threading.Tasks; using Apache.Ignite.Core.Binary; using Apache.Ignite.Core.Impl.Binary; - using Apache.Ignite.Core.Impl.Unmanaged; using Apache.Ignite.Core.Transactions; /// /// Transactions facade. /// - internal class TransactionsImpl : PlatformTarget, ITransactions + internal class TransactionsImpl : PlatformTargetAdapter, ITransactions { /** */ private const int OpCacheConfigParameters = 1; @@ -82,29 +81,19 @@ internal class TransactionsImpl : PlatformTarget, ITransactions /// Initializes a new instance of the class. /// /// Target. - /// Marshaller. /// Local node id. - public TransactionsImpl(IUnmanagedTarget target, Marshaller marsh, - Guid localNodeId) : base(target, marsh) + public TransactionsImpl(IPlatformTargetInternal target, Guid localNodeId) : base(target) { _localNodeId = localNodeId; - TransactionConcurrency concurrency = default(TransactionConcurrency); - TransactionIsolation isolation = default(TransactionIsolation); - TimeSpan timeout = default(TimeSpan); + var res = target.OutStream(OpCacheConfigParameters, reader => Tuple.Create( + (TransactionConcurrency) reader.ReadInt(), + (TransactionIsolation) reader.ReadInt(), + reader.ReadLongAsTimespan())); - DoInOp(OpCacheConfigParameters, stream => - { - var reader = marsh.StartUnmarshal(stream).GetRawReader(); - - concurrency = (TransactionConcurrency) reader.ReadInt(); - isolation = (TransactionIsolation) reader.ReadInt(); - timeout = reader.ReadLongAsTimespan(); - }); - - _dfltConcurrency = concurrency; - _dfltIsolation = isolation; - _dfltTimeout = timeout; + _dfltConcurrency = res.Item1; + _dfltIsolation = res.Item2; + _dfltTimeout = res.Item3; } /** */ diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/IgniteJniNativeMethods.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/IgniteJniNativeMethods.cs index 1720a79ded668..f96157cad5d51 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/IgniteJniNativeMethods.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/IgniteJniNativeMethods.cs @@ -63,6 +63,9 @@ internal static unsafe class IgniteJniNativeMethods [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteTargetInStreamAsync")] public static extern void TargetInStreamAsync(void* ctx, void* target, int opType, long memPtr); + [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteTargetInStreamOutObjectAsync")] + public static extern void* TargetInStreamOutObjectAsync(void* ctx, void* target, int opType, long memPtr); + [DllImport(IgniteUtils.FileIgniteJniDll, EntryPoint = "IgniteAcquire")] public static extern void* Acquire(void* ctx, void* target); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedCallbacks.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedCallbacks.cs index 2400390fcc481..819eda2102fa4 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedCallbacks.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedCallbacks.cs @@ -678,7 +678,11 @@ private long DataStreamerStreamReceiverInvoke(long memPtr, long unused, long unu binaryReceiver.Deserialize(); if (receiver != null) - receiver.Receive(_ignite, new UnmanagedNonReleaseableTarget(_ctx, cache), stream, keepBinary); + { + var target = new PlatformJniTarget(new UnmanagedNonReleaseableTarget(_ctx, cache), + _ignite.Marshaller); + receiver.Receive(_ignite, target, stream, keepBinary); + } return 0; } @@ -1171,9 +1175,9 @@ private long AffinityFunctionInit(long memPtr, long unused, long unused1, void* if (affBase != null) { - var baseFunc0 = UU.Acquire(_ctx, baseFunc); + var baseFunc0 = new PlatformJniTarget(UU.Acquire(_ctx, baseFunc), _ignite.Marshaller); - affBase.SetBaseFunction(new PlatformAffinityFunction(baseFunc0, _ignite.Marshaller)); + affBase.SetBaseFunction(new PlatformAffinityFunction(baseFunc0)); } return _handleRegistry.Allocate(func); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedUtils.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedUtils.cs index a38cf2f9fa6cd..b6e6582d9daea 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedUtils.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/UnmanagedUtils.cs @@ -175,6 +175,13 @@ internal static void TargetInStreamAsync(IUnmanagedTarget target, int opType, lo JNI.TargetInStreamAsync(target.Context, target.Target, opType, memPtr); } + internal static IUnmanagedTarget TargetInStreamOutObjectAsync(IUnmanagedTarget target, int opType, long memPtr) + { + void* res = JNI.TargetInStreamOutObjectAsync(target.Context, target.Target, opType, memPtr); + + return target.ChangeTarget(res); + } + #endregion #region NATIVE METHODS: MISCELANNEOUS diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Interop/IPlatformTarget.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Interop/IPlatformTarget.cs index 621e6041861d8..6f5596a1284b4 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Interop/IPlatformTarget.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Interop/IPlatformTarget.cs @@ -18,6 +18,7 @@ namespace Apache.Ignite.Core.Interop { using System; + using System.Threading; using System.Threading.Tasks; using Apache.Ignite.Core.Binary; @@ -99,5 +100,19 @@ public interface IPlatformTarget /// Task. Task DoOutOpAsync(int type, Action writeAction, Func readAction); + + /// + /// Performs asynchronous operation. + /// + /// Result type + /// Operation type code. + /// Write action (can be null). + /// Read function (can be null). + /// The cancellation token. + /// + /// Task. + /// + Task DoOutOpAsync(int type, Action writeAction, + Func readAction, CancellationToken cancellationToken); } } From b698bbfcaa2056b9792404aef38e427ff323bd57 Mon Sep 17 00:00:00 2001 From: sboikov Date: Fri, 28 Jul 2017 10:25:16 +0300 Subject: [PATCH 054/547] ignite-5858 Fixed affinity initialization on new coordinator (broken in aeb9336b3b161ddfff73f17e41cd453409b84a16). --- .../cache/CacheAffinitySharedManager.java | 2 +- .../cache/GridCachePartitionExchangeManager.java | 13 ++++++++++++- .../preloader/GridDhtPartitionsExchangeFuture.java | 10 +++++++++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java index 5a7f634b7a5a7..51834c5d09784 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java @@ -1461,7 +1461,7 @@ public boolean onServerLeft(final GridDhtPartitionsExchangeFuture fut, boolean c * @throws IgniteCheckedException If failed. * @return Future completed when caches initialization is done. */ - private IgniteInternalFuture initCoordinatorCaches(final GridDhtPartitionsExchangeFuture fut) + public IgniteInternalFuture initCoordinatorCaches(final GridDhtPartitionsExchangeFuture fut) throws IgniteCheckedException { final List> futs = new ArrayList<>(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java index 6a7258fb08d95..f6fa8336b3b7c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java @@ -1745,6 +1745,9 @@ private class ExchangeWorker extends GridWorker { /** Busy flag used as performance optimization to stop current preloading. */ private volatile boolean busy; + /** */ + private boolean crd; + /** * Constructor. */ @@ -1940,7 +1943,15 @@ else if (task instanceof ForceRebalanceExchangeTask) { lastInitializedFut = exchFut; - exchFut.init(); + boolean newCrd = false; + + if (!crd) { + List srvNodes = exchFut.discoCache().serverNodes(); + + crd = newCrd = !srvNodes.isEmpty() && srvNodes.get(0).isLocal(); + } + + exchFut.init(newCrd); int dumpCnt = 0; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 71e41b0e2e9ca..52a74ab91ea5c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -414,9 +414,10 @@ private void leaveBusy() { /** * Starts activity. * + * @param newCrd {@code True} if node become coordinator on this exchange. * @throws IgniteInterruptedCheckedException If interrupted. */ - public void init() throws IgniteInterruptedCheckedException { + public void init(boolean newCrd) throws IgniteInterruptedCheckedException { if (isDone()) return; @@ -489,6 +490,13 @@ else if (msg instanceof SnapshotDiscoveryMessage) { initCachesOnLocalJoin(); } + if (newCrd) { + IgniteInternalFuture fut = cctx.affinity().initCoordinatorCaches(this); + + if (fut != null) + fut.get(); + } + exchange = CU.clientNode(discoEvt.eventNode()) ? onClientNodeEvent(crdNode) : onServerNodeEvent(crdNode); From ab899cf2e383190b71cc8d7940f2880f8b6ee670 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Fri, 28 Jul 2017 11:38:35 +0300 Subject: [PATCH 055/547] IGNITE-5864 .NET: Add missing namespace to LINQPad ComputeExample --- .../dotnet/Apache.Ignite.Core/NuGet/LINQPad/ComputeExample.linq | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/NuGet/LINQPad/ComputeExample.linq b/modules/platforms/dotnet/Apache.Ignite.Core/NuGet/LINQPad/ComputeExample.linq index 75823d15e8d4d..98e3d297bfc9b 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/NuGet/LINQPad/ComputeExample.linq +++ b/modules/platforms/dotnet/Apache.Ignite.Core/NuGet/LINQPad/ComputeExample.linq @@ -5,6 +5,7 @@ Apache.Ignite.Core.Cache.Configuration Apache.Ignite.Core.Cache.Query Apache.Ignite.Core.Compute + Apache.Ignite.Core.Deployment /* From a57a851dd34c296b3b8b919c6f846dfe68192ea1 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Fri, 28 Jul 2017 13:32:26 +0300 Subject: [PATCH 056/547] IGNITE-5865 Fail test with known issue --- .../cache/transactions/TxOptimisticDeadlockDetectionTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java index 15735760eb627..1ee3faa72bd3c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java @@ -148,6 +148,7 @@ public class TxOptimisticDeadlockDetectionTest extends GridCommonAbstractTest { * @throws Exception If failed. */ public void testDeadlocksPartitioned() throws Exception { + fail("IGNITE-5865"); for (CacheWriteSynchronizationMode syncMode : CacheWriteSynchronizationMode.values()) { doTestDeadlocks(createCache(PARTITIONED, syncMode, false), NO_OP_TRANSFORMER); doTestDeadlocks(createCache(PARTITIONED, syncMode, false), WRAPPING_TRANSFORMER); @@ -158,6 +159,7 @@ public void testDeadlocksPartitioned() throws Exception { * @throws Exception If failed. */ public void testDeadlocksPartitionedNear() throws Exception { + fail("IGNITE-5865"); for (CacheWriteSynchronizationMode syncMode : CacheWriteSynchronizationMode.values()) { doTestDeadlocks(createCache(PARTITIONED, syncMode, true), NO_OP_TRANSFORMER); doTestDeadlocks(createCache(PARTITIONED, syncMode, true), WRAPPING_TRANSFORMER); From 1597a186e9d158d12e7e35e9db1158e5252c2e04 Mon Sep 17 00:00:00 2001 From: Igor Sapego Date: Fri, 28 Jul 2017 13:51:25 +0300 Subject: [PATCH 057/547] IGNITE-5758: CPP: Added pointers semantics for primitive types --- .../include/ignite/binary/binary_writer.h | 4 +- .../ignite/impl/binary/binary_reader_impl.h | 50 ++++--- .../ignite/impl/binary/binary_type_impl.h | 67 +++++++++ .../ignite/impl/binary/binary_writer_impl.h | 43 ++++-- .../src/impl/binary/binary_reader_impl.cpp | 63 ++++---- .../src/impl/binary/binary_writer_impl.cpp | 41 +++--- .../src/binary_reader_writer_raw_test.cpp | 36 +++++ .../src/binary_reader_writer_test.cpp | 135 ++++++++++++++---- .../core/include/ignite/impl/ignite_impl.h | 6 - .../cpp/core/src/impl/ignite_impl.cpp | 4 + 10 files changed, 326 insertions(+), 123 deletions(-) diff --git a/modules/platforms/cpp/binary/include/ignite/binary/binary_writer.h b/modules/platforms/cpp/binary/include/ignite/binary/binary_writer.h index 1489494f34cdf..e609591f72661 100644 --- a/modules/platforms/cpp/binary/include/ignite/binary/binary_writer.h +++ b/modules/platforms/cpp/binary/include/ignite/binary/binary_writer.h @@ -58,7 +58,7 @@ namespace ignite * * @param impl Implementation. */ - BinaryWriter(ignite::impl::binary::BinaryWriterImpl* impl); + BinaryWriter(impl::binary::BinaryWriterImpl* impl); /** * Write 8-byte signed integer. Maps to "byte" type in Java. @@ -337,7 +337,7 @@ namespace ignite * Start collection write. * * @param fieldName Field name. - * @param type Collection type. + * @param typ Collection type. * @return Collection writer. */ template diff --git a/modules/platforms/cpp/binary/include/ignite/impl/binary/binary_reader_impl.h b/modules/platforms/cpp/binary/include/ignite/impl/binary/binary_reader_impl.h index 5621c0d513f70..4a0e2d43383f2 100644 --- a/modules/platforms/cpp/binary/include/ignite/impl/binary/binary_reader_impl.h +++ b/modules/platforms/cpp/binary/include/ignite/impl/binary/binary_reader_impl.h @@ -883,6 +883,17 @@ namespace ignite */ template T ReadTopObject() + { + return ignite::binary::ReadHelper::Read(*this); + } + + /** + * Read object. + * + * @return Read object. + */ + template + void ReadTopObject0(T& res) { int32_t pos = stream->Position(); int8_t hdr = stream->ReadInt8(); @@ -891,7 +902,9 @@ namespace ignite { case IGNITE_HDR_NULL: { - return GetNull(); + res = GetNull(); + + return; } case IGNITE_HDR_HND: @@ -908,11 +921,11 @@ namespace ignite stream->Position(curPos + portOff); // Position stream right on the object. - T val = ReadTopObject(); + ReadTopObject0(res); stream->Position(curPos + portLen + 4); // Position stream after binary. - return val; + return; } case IGNITE_HDR_FULL: @@ -985,12 +998,11 @@ namespace ignite footerBegin, footerEnd, schemaType); ignite::binary::BinaryReader reader(&readerImpl); - T val; - BType::Read(reader, val); + BType::Read(reader, res); stream->Position(pos + len); - return val; + return; } default: @@ -1407,43 +1419,43 @@ namespace ignite }; template<> - int8_t IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject(); + void IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject0(int8_t& res); template<> - bool IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject(); + void IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject0(bool& res); template<> - int16_t IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject(); + void IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject0(int16_t& res); template<> - uint16_t IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject(); + void IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject0(uint16_t& res); template<> - int32_t IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject(); + void IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject0(int32_t& res); template<> - int64_t IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject(); + void IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject0(int64_t& res); template<> - float IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject(); + void IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject0(float& res); template<> - double IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject(); + void IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject0(double& res); template<> - Guid IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject(); + void IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject0(Guid& res); template<> - Date IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject(); + void IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject0(Date& res); template<> - Timestamp IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject(); + void IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject0(Timestamp& res); template<> - Time IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject