From efbac32e650d603e18ea405bee626589ac237f61 Mon Sep 17 00:00:00 2001 From: Vyacheslav Daradur Date: Wed, 9 Aug 2017 17:05:29 +0300 Subject: [PATCH 1/7] IGNITE-5732 Provide API to test compatibility with old releases (#20) --- modules/compatibility/README.txt | 5 + modules/compatibility/licenses/apache-2.0.txt | 202 +++++++++++++++ modules/compatibility/pom.xml | 94 +++++++ .../ignite/compatibility/package-info.java | 22 ++ .../spi/MultiVersionClusterTest.java | 149 +++++++++++ .../compatibility/spi/package-info.java | 22 ++ .../CompatibilityTestIgniteNodeRunner.java | 129 +++++++++ .../junits/CompatibilityTestsFacade.java | 46 ++++ .../IgniteCompatibilityAbstractTest.java | 244 ++++++++++++++++++ .../logger/ListenedGridTestLog4jLogger.java | 90 +++++++ .../testframework/junits/package-info.java | 22 ++ .../testframework/package-info.java | 22 ++ .../plugins/NoNodeValidationProcessor.java | 42 +++ .../TestCompatibilityPluginProvider.java | 137 ++++++++++ .../testframework/plugins/package-info.java | 22 ++ .../testframework/util/MavenUtils.java | 208 +++++++++++++++ .../testframework/util/package-info.java | 22 ++ .../IgniteCompatibilityBasicTestSuite.java | 38 +++ .../testsuites/package-info.java | 22 ++ .../org.apache.ignite.plugin.PluginProvider | 1 + .../junits/multijvm/IgniteProcessProxy.java | 90 +++++-- pom.xml | 10 + 22 files changed, 1617 insertions(+), 22 deletions(-) create mode 100644 modules/compatibility/README.txt create mode 100644 modules/compatibility/licenses/apache-2.0.txt create mode 100644 modules/compatibility/pom.xml create mode 100644 modules/compatibility/src/test/java/org/apache/ignite/compatibility/package-info.java create mode 100644 modules/compatibility/src/test/java/org/apache/ignite/compatibility/spi/MultiVersionClusterTest.java create mode 100644 modules/compatibility/src/test/java/org/apache/ignite/compatibility/spi/package-info.java create mode 100644 modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/CompatibilityTestIgniteNodeRunner.java create mode 100644 modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/CompatibilityTestsFacade.java create mode 100644 modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/IgniteCompatibilityAbstractTest.java create mode 100644 modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/logger/ListenedGridTestLog4jLogger.java create mode 100644 modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/package-info.java create mode 100644 modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/package-info.java create mode 100644 modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/plugins/NoNodeValidationProcessor.java create mode 100644 modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/plugins/TestCompatibilityPluginProvider.java create mode 100644 modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/plugins/package-info.java create mode 100644 modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/util/MavenUtils.java create mode 100644 modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/util/package-info.java create mode 100644 modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java create mode 100644 modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/package-info.java create mode 100644 modules/compatibility/src/test/resources/META-INF/services/org.apache.ignite.plugin.PluginProvider diff --git a/modules/compatibility/README.txt b/modules/compatibility/README.txt new file mode 100644 index 0000000000000..f2743ecdad754 --- /dev/null +++ b/modules/compatibility/README.txt @@ -0,0 +1,5 @@ +Apache Ignite Compatibility Tests +------------------------ + +Special module contains testing framework which provides methods for testing backward compatibility. +It allows to start Ignite-cluster in multiversion mode. It means that nodes with different build version is allowed to join topology. \ No newline at end of file diff --git a/modules/compatibility/licenses/apache-2.0.txt b/modules/compatibility/licenses/apache-2.0.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/modules/compatibility/licenses/apache-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/modules/compatibility/pom.xml b/modules/compatibility/pom.xml new file mode 100644 index 0000000000000..6d5381bff0130 --- /dev/null +++ b/modules/compatibility/pom.xml @@ -0,0 +1,94 @@ + + + + + + + 4.0.0 + + + org.apache.ignite + ignite-parent + 1 + ../../parent + + + ignite-compatibility + 2.2.0-SNAPSHOT + http://ignite.apache.org + + + + org.apache.ignite + ignite-core + ${project.version} + + + + com.google.guava + guava + ${guava.version} + + + + org.apache.ignite + ignite-core + ${project.version} + test-jar + test + + + + com.thoughtworks.xstream + xstream + 1.4.8 + test + + + + org.springframework + spring-core + ${spring.version} + test + + + + log4j + log4j + test + + + + org.springframework + spring-beans + ${spring.version} + test + + + + org.springframework + spring-context + ${spring.version} + test + + + diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/package-info.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/package-info.java new file mode 100644 index 0000000000000..73d942366fdfe --- /dev/null +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + * Contains compatibility tests or compatibility test related classes and interfaces. + */ +package org.apache.ignite.compatibility; \ No newline at end of file diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/spi/MultiVersionClusterTest.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/spi/MultiVersionClusterTest.java new file mode 100644 index 0000000000000..d2769d45c01dc --- /dev/null +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/spi/MultiVersionClusterTest.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compatibility.spi; + +import org.apache.ignite.compatibility.testframework.junits.IgniteCompatibilityAbstractTest; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.GridCacheAbstractFullApiSelfTest; +import org.apache.ignite.lang.IgniteInClosure; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; + +/** */ +public class MultiVersionClusterTest extends IgniteCompatibilityAbstractTest { + /** */ + public void testJoinMultiVersionTopologyLocalFirst() throws Exception { + try { + IgniteEx ignite = startGrid(0); + + assertEquals(1, topologyVersion(ignite)); + + startGrid("testMultiVersion", "2.1.0", new PostConfigurationClosure()); + + assertEquals(2, topologyVersion(ignite)); + } + finally { + stopAllGrids(); + } + } + + /** */ + public void testJoinMultiVersionTopologyLocalFirst2() throws Exception { + try { + IgniteEx ignite = startGrid(0); + + assertEquals(1, topologyVersion(ignite)); + + startGrid(1, "2.1.0", new PostConfigurationClosure()); + + assertEquals(2, topologyVersion(ignite)); + + startGrid(2, "2.1.0", new PostConfigurationClosure()); + + assertEquals(3, topologyVersion(ignite)); + + startGrid(3, "2.1.0", new PostConfigurationClosure()); + + assertEquals(4, topologyVersion(ignite)); + } + finally { + stopAllGrids(); + } + } + + /** */ + public void testJoinMultiVersionTopologyRemoteFirst() throws Exception { + try { + startGrid(1, "2.1.0", new PostConfigurationClosure()); + + IgniteEx ignite = startGrid(0); + + assertEquals(2, topologyVersion(ignite)); + } + finally { + stopAllGrids(); + } + } + + /** */ + public void testJoinMultiVersionTopologyRemoteFirst2() throws Exception { + try { + startGrid(1, "2.1.0", new PostConfigurationClosure()); + + IgniteEx ignite = startGrid(0); + + assertEquals(2, topologyVersion(ignite)); + + startGrid(3, "2.1.0", new PostConfigurationClosure()); + + assertEquals(3, topologyVersion(ignite)); + + startGrid(4); + + assertEquals(4, topologyVersion(ignite)); + } + finally { + stopAllGrids(); + } + } + + /** */ + public void testJoinMultiVersionTopologyRemoteFirst3() throws Exception { + try { + startGrid(1, "2.1.0", new PostConfigurationClosure()); + + startGrid(2, "2.1.0", new PostConfigurationClosure()); + + IgniteEx ignite = startGrid(0); + + assertEquals(3, topologyVersion(ignite)); + + startGrid(3, "2.1.0", new PostConfigurationClosure()); + + assertEquals(4, topologyVersion(ignite)); + + startGrid(4, "2.1.0", new PostConfigurationClosure()); + + assertEquals(5, topologyVersion(ignite)); + } + finally { + stopAllGrids(); + } + } + + /** */ + private static class PostConfigurationClosure implements IgniteInClosure { + /** {@inheritDoc} */ + @Override public void apply(IgniteConfiguration cfg) { + cfg.setLocalHost("127.0.0.1"); + + TcpDiscoverySpi disco = new TcpDiscoverySpi(); + disco.setIpFinder(GridCacheAbstractFullApiSelfTest.LOCAL_IP_FINDER); + + cfg.setDiscoverySpi(disco); + } + } + + /** + * @param ignite Ignite instance. + * @return Topology version. + */ + private static long topologyVersion(IgniteEx ignite) { + return ignite.context().discovery().topologyVersion(); + } +} diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/spi/package-info.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/spi/package-info.java new file mode 100644 index 0000000000000..fec03de3a842a --- /dev/null +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/spi/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + * Contains compatibility tests or compatibility test related classes and interfaces. + */ +package org.apache.ignite.compatibility.spi; \ No newline at end of file diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/CompatibilityTestIgniteNodeRunner.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/CompatibilityTestIgniteNodeRunner.java new file mode 100644 index 0000000000000..8a6ff6d9deb3a --- /dev/null +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/CompatibilityTestIgniteNodeRunner.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compatibility.testframework.junits; + +import com.thoughtworks.xstream.XStream; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.UUID; +import org.apache.ignite.Ignite; +import org.apache.ignite.Ignition; +import org.apache.ignite.compatibility.testframework.plugins.TestCompatibilityPluginProvider; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.util.GridJavaProcess; +import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteInClosure; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.multijvm.IgniteNodeRunner; + +/** + * Run Ignite node. + */ +public class CompatibilityTestIgniteNodeRunner extends IgniteNodeRunner { + /** */ + private static final String IGNITE_CLOSURE_FILE = System.getProperty("java.io.tmpdir") + + File.separator + "igniteClosure.tmp_"; + + /** + * Starts {@link Ignite} with test's default configuration. + * + * @param args Arguments. + * @throws Exception In case of an error. + */ + public static void main(String[] args) throws Exception { + X.println(GridJavaProcess.PID_MSG_PREFIX + U.jvmPid()); + + X.println("Starting Ignite Node... Args=" + Arrays.toString(args)); + + if (args.length < 3) + throw new IllegalArgumentException("Three arguments expected: [path/to/closure/file] [ignite-instance-name] [node-uuid]"); + + TestCompatibilityPluginProvider.enable(); + + IgniteConfiguration cfg = CompatibilityTestsFacade.getConfiguration(); + + IgniteInClosure clos = readClosureFromFileAndDelete(args[0]); + + clos.apply(cfg); + + // Ignite instance name and id must be set according to arguments + // it's used for nodes managing: start, stop etc. + cfg.setIgniteInstanceName(args[1]); + cfg.setNodeId(UUID.fromString(args[2])); + + Ignite ignite = Ignition.start(cfg); + + GridTestUtils.setFieldValue(new IgniteNodeRunner(), "ignite", ignite); + } + + /** + * Stores {@link IgniteInClosure} to file as xml. + * + * @param clos IgniteInClosure. + * @return A name of file where the closure was stored. + * @throws IOException In case of an error. + * @see #readClosureFromFileAndDelete(String) + */ + public static String storeToFile(IgniteInClosure clos) throws IOException { + String fileName = IGNITE_CLOSURE_FILE + clos.hashCode(); + + storeToFile(clos, fileName); + + return fileName; + } + + /** + * Stores {@link IgniteInClosure} to file as xml. + * + * @param clos IgniteInClosure. + * @param fileName A name of file where the closure was stored. + * @throws IOException In case of an error. + * @see #readClosureFromFileAndDelete(String) + */ + public static void storeToFile(IgniteInClosure clos, String fileName) throws IOException { + try (BufferedWriter writer = Files.newBufferedWriter(Paths.get(fileName), StandardCharsets.UTF_8)) { + new XStream().toXML(clos, writer); + } + } + + /** + * Reads closure from given file name and delete the file after. + * + * @param closFileName Closure file name. + * @return IgniteInClosure for post-configuration. + * @throws IOException In case of an error. + * @see #storeToFile(IgniteInClosure, String) + */ + @SuppressWarnings("unchecked") + public static IgniteInClosure readClosureFromFileAndDelete( + String closFileName) throws IOException { + try (BufferedReader closReader = Files.newBufferedReader(Paths.get(closFileName), StandardCharsets.UTF_8)) { + return (IgniteInClosure)new XStream().fromXML(closReader); + } + finally { + new File(closFileName).delete(); + } + } +} diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/CompatibilityTestsFacade.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/CompatibilityTestsFacade.java new file mode 100644 index 0000000000000..6049314654fb9 --- /dev/null +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/CompatibilityTestsFacade.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compatibility.testframework.junits; + +import org.apache.ignite.configuration.IgniteConfiguration; + +/** + * Provides useful methods for compatibility tests. + */ +public class CompatibilityTestsFacade { + /** */ + private static final TestsMethodsFacade DELEGATE = new TestsMethodsFacade(); + + /** + * Gets default tests configuration. + * + * @return Default tests Ignite configuration. + * @throws Exception In case of an error. + */ + public static IgniteConfiguration getConfiguration() throws Exception { + return DELEGATE.getConfiguration(); + } + + /** */ + private static class TestsMethodsFacade extends IgniteCompatibilityAbstractTest { + /** {@inheritDoc} */ + @Override public IgniteConfiguration getConfiguration() throws Exception { + return super.getConfiguration(); + } + } +} diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/IgniteCompatibilityAbstractTest.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/IgniteCompatibilityAbstractTest.java new file mode 100644 index 0000000000000..bad2005a46075 --- /dev/null +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/IgniteCompatibilityAbstractTest.java @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compatibility.testframework.junits; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.compatibility.testframework.junits.logger.ListenedGridTestLog4jLogger; +import org.apache.ignite.compatibility.testframework.plugins.TestCompatibilityPluginProvider; +import org.apache.ignite.compatibility.testframework.util.MavenUtils; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.resource.GridSpringResourceContext; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteInClosure; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.testframework.junits.multijvm.IgniteProcessProxy; +import org.jetbrains.annotations.Nullable; + +/** + * Super class for all compatibility tests. + */ +public abstract class IgniteCompatibilityAbstractTest extends GridCommonAbstractTest { + /** Local JVM Ignite node. */ + protected Ignite locJvmInstance = null; + + /** Remote JVM Ignite instance. */ + protected Ignite rmJvmInstance = null; + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + TestCompatibilityPluginProvider.enable(); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + super.afterTestsStopped(); + + TestCompatibilityPluginProvider.disable(); + } + + /** {@inheritDoc} */ + @Override protected boolean isMultiJvm() { + return true; + } + + /** + * Starts new grid of given version and index in separate JVM. + * + * Uses an ignite-core artifact in the Maven local repository, if it isn't exists there, it will be downloaded and + * stored via Maven. + * + * @param idx Index of the grid to start. + * @param ver Ignite version. + * @param clos IgniteInClosure for post-configuration. + * @return Started grid. + * @throws Exception If failed. + */ + protected IgniteEx startGrid(int idx, String ver, IgniteInClosure clos) throws Exception { + return startGrid(getTestIgniteInstanceName(idx), ver, clos); + } + + /** + * Starts new grid of given version and name in separate JVM. + * + * Uses an ignite-core artifact in the Maven local repository, if it isn't exists there, it will be downloaded and + * stored via Maven. + * + * @param igniteInstanceName Instance name. + * @param ver Ignite version. + * @param clos IgniteInClosure for post-configuration. + * @return Started grid. + * @throws Exception If failed. + */ + protected IgniteEx startGrid(final String igniteInstanceName, final String ver, + IgniteInClosure clos) throws Exception { + assert isMultiJvm() : "MultiJvm mode must be switched on for the node stop properly."; + + assert !igniteInstanceName.equals(getTestIgniteInstanceName(0)) : "Use default instance name for local nodes only."; + + final String closPath = CompatibilityTestIgniteNodeRunner.storeToFile(clos); + + final IgniteConfiguration cfg = getConfiguration(igniteInstanceName); // stub - won't be used at node startup + + IgniteProcessProxy ignite = new IgniteProcessProxy(cfg, log, locJvmInstance, true) { + @Override protected IgniteLogger logger(IgniteLogger log, Object ctgr) { + return ListenedGridTestLog4jLogger.createLogger(ctgr); + } + + @Override protected String igniteNodeRunnerClassName() throws Exception { + return CompatibilityTestIgniteNodeRunner.class.getCanonicalName(); + } + + @Override protected String params(IgniteConfiguration cfg, boolean resetDiscovery) throws Exception { + return closPath + " " + igniteInstanceName + " " + getId(); + } + + @Override protected Collection filteredJvmArgs() throws Exception { + Collection filteredJvmArgs = new ArrayList<>(); + + filteredJvmArgs.add("-ea"); + + for (String arg : U.jvmArgs()) { + if (arg.startsWith("-Xmx") || arg.startsWith("-Xms")) + filteredJvmArgs.add(arg); + } + + String classPath = System.getProperty("java.class.path"); + + String[] paths = classPath.split(File.pathSeparator); + + StringBuilder pathBuilder = new StringBuilder(); + + String corePathTemplate = "ignite.modules.core.target.classes".replace(".", File.separator); + String coreTestsPathTemplate = "ignite.modules.core.target.test-classes".replace(".", File.separator); + + for (String path : paths) { + if (!path.contains(corePathTemplate) && !path.contains(coreTestsPathTemplate)) + pathBuilder.append(path).append(File.pathSeparator); + } + + String pathToArtifact = MavenUtils.getPathToIgniteCoreArtifact(ver); + pathBuilder.append(pathToArtifact).append(File.pathSeparator); + + String pathToTestsArtifact = MavenUtils.getPathToIgniteCoreArtifact(ver, "tests"); + pathBuilder.append(pathToTestsArtifact).append(File.pathSeparator); + + filteredJvmArgs.add("-cp"); + filteredJvmArgs.add(pathBuilder.toString()); + + return filteredJvmArgs; + } + }; + + if (rmJvmInstance == null) + rmJvmInstance = ignite; + + if (locJvmInstance == null) { + CountDownLatch nodeJoinedLatch = new CountDownLatch(1); + + UUID nodeId = ignite.getId(); + + ListenedGridTestLog4jLogger log = (ListenedGridTestLog4jLogger)rmJvmInstance.log(); + + log.addListener(nodeId, new LoggedJoinNodeClosure(nodeJoinedLatch, nodeId)); + + assert nodeJoinedLatch.await(30, TimeUnit.SECONDS) : "Node has not joined [id=" + nodeId + "]"; + + log.removeListener(nodeId); + } + + return ignite; + } + + /** {@inheritDoc} */ + @Override protected Ignite startGrid(String igniteInstanceName, IgniteConfiguration cfg, + GridSpringResourceContext ctx) throws Exception { + Ignite ignite; + + // if started node isn't first node in the local JVM then it was checked earlier for join to topology + // in IgniteProcessProxy constructor. + if (locJvmInstance == null && rmJvmInstance != null) { + CountDownLatch nodeJoinedLatch = new CountDownLatch(1); + + UUID nodeId = cfg.getNodeId(); + + ListenedGridTestLog4jLogger log = (ListenedGridTestLog4jLogger)rmJvmInstance.log(); + + log.addListener(nodeId, new LoggedJoinNodeClosure(nodeJoinedLatch, nodeId)); + + ignite = super.startGrid(igniteInstanceName, cfg, ctx); + + assert ignite.configuration().getNodeId() == cfg.getNodeId() : "Started node have unexpected node id."; + + assert nodeJoinedLatch.await(30, TimeUnit.SECONDS) : "Node has not joined [id=" + nodeId + "]"; + + log.removeListener(nodeId); + } + else + ignite = super.startGrid(igniteInstanceName, cfg, ctx); + + if (locJvmInstance == null && !isRemoteJvm(igniteInstanceName)) + locJvmInstance = ignite; + + return ignite; + } + + /** {@inheritDoc} */ + @Override protected void stopGrid(@Nullable String igniteInstanceName, boolean cancel, boolean awaitTop) { + if (isRemoteJvm(igniteInstanceName)) + throw new UnsupportedOperationException("Operation isn't supported yet for remotes nodes, use stopAllGrids() instead."); + else { + super.stopGrid(igniteInstanceName, cancel, awaitTop); + + locJvmInstance = null; + } + } + + /** */ + protected static class LoggedJoinNodeClosure implements IgniteInClosure { + /** Node joined latch. */ + private CountDownLatch nodeJoinedLatch; + + /** Pattern for comparing. */ + private String pattern; + + /** + * @param nodeJoinedLatch Node joined latch. + * @param nodeId Expected node id. + */ + public LoggedJoinNodeClosure(CountDownLatch nodeJoinedLatch, UUID nodeId) { + this.nodeJoinedLatch = nodeJoinedLatch; + this.pattern = "evt=NODE_JOINED, node=" + nodeId; + } + + /** {@inheritDoc} */ + @Override public void apply(String s) { + if (s.contains(pattern) && nodeJoinedLatch.getCount() > 0) + nodeJoinedLatch.countDown(); + } + } +} diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/logger/ListenedGridTestLog4jLogger.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/logger/ListenedGridTestLog4jLogger.java new file mode 100644 index 0000000000000..74a2f3fd52d51 --- /dev/null +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/logger/ListenedGridTestLog4jLogger.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compatibility.testframework.junits.logger; + +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import org.apache.ignite.lang.IgniteInClosure; +import org.apache.ignite.testframework.junits.logger.GridTestLog4jLogger; +import org.apache.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jsr166.ConcurrentHashMap8; + +/** + * Listened version of {@link GridTestLog4jLogger} logger. Provides methods of registering and notifying listeners. + */ +public class ListenedGridTestLog4jLogger extends GridTestLog4jLogger { + /** Listeners. */ + private final ConcurrentMap> lsnrs = new ConcurrentHashMap8<>(); + + /** {@inheritDoc} */ + public ListenedGridTestLog4jLogger(Logger impl) { + super(impl); + } + + /** + * Creates new logger instance with given category. + * + * @param ctgr Category. + * @return Initiated logger. + */ + public static GridTestLog4jLogger createLogger(Object ctgr) { + return new ListenedGridTestLog4jLogger(ctgr == null ? Logger.getRootLogger() : + ctgr instanceof Class ? Logger.getLogger(((Class)ctgr).getName()) : + Logger.getLogger(ctgr.toString())); + } + + /** {@inheritDoc} */ + @Override public void info(String msg) { + notifyListeners(msg); + + super.info(msg); + } + + /** + * Notifies registered listeners. + * + * @param msg Messaged. + */ + protected void notifyListeners(String msg) { + for (IgniteInClosure lsnr : lsnrs.values()) + lsnr.apply(msg); + } + + /** + * Adds listener. + * + * @param key Key. + * @param lsnr Listener. + * @return The previous value associated with the specified key, or null if there was no mapping for the key. + */ + public IgniteInClosure addListener(@NotNull UUID key, @NotNull IgniteInClosure lsnr) { + return lsnrs.putIfAbsent(key, lsnr); + } + + /** + * Removes listeners. + * + * @param key Key. + * @return Returns the value to which this map previously associated the key, or null if the map contained no + * mapping for the key. + */ + public IgniteInClosure removeListener(@NotNull UUID key) { + return lsnrs.remove(key); + } +} diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/package-info.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/package-info.java new file mode 100644 index 0000000000000..996729f7e1e02 --- /dev/null +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + * Contains compatibility test framework related classes and interfaces. + */ +package org.apache.ignite.compatibility.testframework.junits; \ No newline at end of file diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/package-info.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/package-info.java new file mode 100644 index 0000000000000..81ec1b6ddb0c4 --- /dev/null +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + * Contains compatibility test framework related classes and interfaces. + */ +package org.apache.ignite.compatibility.testframework; \ No newline at end of file diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/plugins/NoNodeValidationProcessor.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/plugins/NoNodeValidationProcessor.java new file mode 100644 index 0000000000000..9ad59540c747f --- /dev/null +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/plugins/NoNodeValidationProcessor.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compatibility.testframework.plugins; + +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.processors.GridProcessorAdapter; +import org.apache.ignite.internal.processors.nodevalidation.DiscoveryNodeValidationProcessor; +import org.apache.ignite.spi.IgniteNodeValidationResult; +import org.jetbrains.annotations.Nullable; + +/** + * Disabled node validation. + */ +public class NoNodeValidationProcessor extends GridProcessorAdapter implements DiscoveryNodeValidationProcessor { + /** + * @param ctx Kernal context. + */ + public NoNodeValidationProcessor(GridKernalContext ctx) { + super(ctx); + } + + /** {@inheritDoc} */ + @Nullable @Override public IgniteNodeValidationResult validateNode(ClusterNode node) { + return null; + } +} diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/plugins/TestCompatibilityPluginProvider.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/plugins/TestCompatibilityPluginProvider.java new file mode 100644 index 0000000000000..edb9614e48c49 --- /dev/null +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/plugins/TestCompatibilityPluginProvider.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compatibility.testframework.plugins; + +import java.io.Serializable; +import java.util.UUID; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteKernal; +import org.apache.ignite.internal.IgniteNodeAttributes; +import org.apache.ignite.internal.processors.nodevalidation.DiscoveryNodeValidationProcessor; +import org.apache.ignite.plugin.CachePluginContext; +import org.apache.ignite.plugin.CachePluginProvider; +import org.apache.ignite.plugin.ExtensionRegistry; +import org.apache.ignite.plugin.IgnitePlugin; +import org.apache.ignite.plugin.PluginContext; +import org.apache.ignite.plugin.PluginProvider; +import org.apache.ignite.plugin.PluginValidationException; +import org.jetbrains.annotations.Nullable; + +/** + * Creates {@link NoNodeValidationProcessor} to allow for nodes with different + * versions {@link IgniteNodeAttributes#ATTR_BUILD_VER} join to topology. + */ +public class TestCompatibilityPluginProvider implements PluginProvider { + /** */ + private static volatile boolean enabled; + + /** */ + private GridKernalContext kCtx; + + /** */ + public static boolean isEnabled() { + return enabled; + } + + /** */ + public static void enable() { + enabled = true; + } + + /** */ + public static void disable() { + enabled = false; + } + + /** {@inheritDoc} */ + @Override public String name() { + return "TestCompatibilityPlugin"; + } + + /** {@inheritDoc} */ + @Override public String version() { + return "1.0"; + } + + /** {@inheritDoc} */ + @Override public String copyright() { + return ""; + } + + /** {@inheritDoc} */ + @Override public void initExtensions(PluginContext ctx, ExtensionRegistry registry) throws IgniteCheckedException { + kCtx = ((IgniteKernal)ctx.grid()).context(); + } + + /** {@inheritDoc} */ + @Override public CachePluginProvider createCacheProvider(CachePluginContext ctx) { + return null; + } + + /** {@inheritDoc} */ + @Override public void start(PluginContext ctx) throws IgniteCheckedException { + // No-op. + } + + /** {@inheritDoc} */ + @Override public void stop(boolean cancel) throws IgniteCheckedException { + // No-op. + } + + /** {@inheritDoc} */ + @Override public void onIgniteStart() throws IgniteCheckedException { + // No-op. + } + + /** {@inheritDoc} */ + @Override public void onIgniteStop(boolean cancel) { + // No-op. + } + + /** {@inheritDoc} */ + @Nullable @Override public Serializable provideDiscoveryData(UUID nodeId) { + return null; + } + + /** {@inheritDoc} */ + @Override public void receiveDiscoveryData(UUID nodeId, Serializable data) { + // No-op. + } + + /** {@inheritDoc} */ + @Override public void validateNewNode(ClusterNode node) throws PluginValidationException { + // No-op. + } + + /** {@inheritDoc} */ + @Nullable @Override public Object createComponent(PluginContext ctx, Class cls) { + if (enabled && DiscoveryNodeValidationProcessor.class == cls) + return new NoNodeValidationProcessor(kCtx); + + return null; + } + + /** {@inheritDoc} */ + @Override public IgnitePlugin plugin() { + return new IgnitePlugin() { + // No-op. + }; + } +} diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/plugins/package-info.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/plugins/package-info.java new file mode 100644 index 0000000000000..05ba5c2466974 --- /dev/null +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/plugins/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + * Contains compatibility test framework related classes and interfaces. + */ +package org.apache.ignite.compatibility.testframework.plugins; \ No newline at end of file diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/util/MavenUtils.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/util/MavenUtils.java new file mode 100644 index 0000000000000..fe73e48555c1d --- /dev/null +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/util/MavenUtils.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compatibility.testframework.util; + +import com.google.common.base.Charsets; +import com.google.common.io.CharStreams; +import java.io.File; +import java.io.InputStreamReader; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.concurrent.Callable; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * Provides some useful methods to work with Maven. + */ +public class MavenUtils { + /** Path to Maven local repository. For caching. */ + private static String locRepPath = null; + + /** + * Gets a path to an artifact with given version and groupId=org.apache.ignite and artifactId=ignite-core. + * + * At first, artifact is looked for in the Maven local repository, if it isn't exists there, it will be downloaded + * and stored via Maven. + * + * @param ver Version of ignite-core artifact. + * @return Path to the artifact. + * @throws Exception In case of an error. + * @see #getPathToArtifact(String) + */ + public static String getPathToIgniteCoreArtifact(@NotNull String ver) throws Exception { + return getPathToIgniteCoreArtifact(ver, null); + } + + /** + * Gets a path to an artifact with given version and groupId=org.apache.ignite and artifactId=ignite-core. + * + * At first, artifact is looked for in the Maven local repository, if it isn't exists there, it will be downloaded + * and stored via Maven. + * + * @param ver Version of ignite-core artifact. + * @param classifier Artifact classifier. + * @return Path to the artifact. + * @throws Exception In case of an error. + * @see #getPathToArtifact(String) + */ + public static String getPathToIgniteCoreArtifact(@NotNull String ver, + @Nullable String classifier) throws Exception { + String artifact = "org.apache.ignite:ignite-core:" + ver; + + if (classifier != null) + artifact += ":jar:" + classifier; + + return getPathToArtifact(artifact); + } + + /** + * Gets a path to an artifact with given identifier. + * + * At first, artifact is looked for in the Maven local repository, if it isn't exists there, it will be downloaded + * and stored via Maven. + * + * @param artifact Artifact identifier, must match pattern [groupId:artifactId:version[:packaging[:classifier]]]. + * @return Path to the artifact. + * @throws Exception In case of an error. + */ + public static String getPathToArtifact(@NotNull String artifact) throws Exception { + String[] names = artifact.split(":"); + + assert names.length >= 3; + + String groupId = names[0]; + String artifactId = names[1]; + String version = names[2]; + String packaging = names.length > 3 ? names[3] : null; + String classifier = names.length > 4 ? names[4] : null; + + String jarFileName = String.format("%s-%s%s.%s", + artifactId, + version, + (classifier == null ? "" : "-" + classifier), + (packaging == null ? "jar" : packaging) + ); + + String pathToArtifact = getMavenLocalRepositoryPath() + File.separator + + groupId.replace(".", File.separator) + File.separator + + artifactId.replace(".", File.separator) + File.separator + + version + File.separator + jarFileName; + + if (Files.notExists(Paths.get(pathToArtifact))) + downloadArtifact(artifact); + + return pathToArtifact; + } + + /** + * Gets a path to the Maven local repository. + * + * @return Path to the Maven local repository. + * @throws Exception In case of an error. + */ + public static String getMavenLocalRepositoryPath() throws Exception { + if (locRepPath == null) + locRepPath = defineMavenLocalRepositoryPath(); + + return locRepPath; + } + + /** + * Defines a path to the Maven local repository. + * + * @return Path to the Maven local repository. + * @throws Exception In case of an error. + */ + private static String defineMavenLocalRepositoryPath() throws Exception { + String output = exec("mvn help:effective-settings"); + + int endTagPos = output.indexOf(""); + + assert endTagPos >= 0 : "Couldn't define path to Maven local repository"; + + return output.substring(output.lastIndexOf(">", endTagPos) + 1, endTagPos); + } + + /** + * Downloads and stores in local repository an artifact with given identifier. + * + * @param artifact Artifact identifier, must match pattern [groupId:artifactId:version]. + * @throws Exception In case of an error. + */ + private static void downloadArtifact(String artifact) throws Exception { + X.println("Downloading artifact... Identifier: " + artifact); + + exec("mvn dependency:get -Dartifact=" + artifact); + + X.println("Download is finished"); + } + + /** + * Executes given command in operation system. + * + * @param cmd Command to execute. + * @return Output of result of executed command. + * @throws Exception In case of an error. + */ + private static String exec(String cmd) throws Exception { + ProcessBuilder pb = new ProcessBuilder(); + pb.redirectErrorStream(true); + + pb.command(U.isWindows() ? + new String[] {"cmd", "/c", cmd} : + new String[] {"/bin/bash", "-c", cmd}); + + final Process p = pb.start(); + + Future fut = Executors.newSingleThreadExecutor().submit(new Callable() { + @Override public String call() throws Exception { + String output = CharStreams.toString(new InputStreamReader(p.getInputStream(), Charsets.UTF_8)); + + p.waitFor(); + + return output; + } + }); + + String output = null; + + try { + output = fut.get(5, TimeUnit.MINUTES); + + int exitVal = p.exitValue(); + + if (exitVal != 0) + throw new Exception(String.format("Abnormal exit value of %s for pid %s", exitVal, U.jvmPid())); + + return output; + } + catch (Exception e) { + p.destroy(); + + X.printerrln("Command='" + cmd + "' couldn't be executed: " + output, Charsets.UTF_8, e); + + throw e; + } + } +} diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/util/package-info.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/util/package-info.java new file mode 100644 index 0000000000000..d3bee94387075 --- /dev/null +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/util/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + * Contains compatibility test framework related classes and interfaces. + */ +package org.apache.ignite.compatibility.testframework.util; \ No newline at end of file diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java new file mode 100644 index 0000000000000..78e769a5ad80f --- /dev/null +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compatibility.testsuites; + +import junit.framework.TestSuite; +import org.apache.ignite.compatibility.spi.MultiVersionClusterTest; + +/** + * Compatibility tests basic test suite. + */ +public class IgniteCompatibilityBasicTestSuite { + /** + * @return Test suite. + * @throws Exception In case of an error. + */ + public static TestSuite suite() throws Exception { + TestSuite suite = new TestSuite("Ignite Compatibility Basic Test Suite"); + + suite.addTestSuite(MultiVersionClusterTest.class); + + return suite; + } +} diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/package-info.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/package-info.java new file mode 100644 index 0000000000000..5c4efc99ca1e5 --- /dev/null +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + * Contains test-suites which contains compatibility tests. + */ +package org.apache.ignite.compatibility.testsuites; \ No newline at end of file diff --git a/modules/compatibility/src/test/resources/META-INF/services/org.apache.ignite.plugin.PluginProvider b/modules/compatibility/src/test/resources/META-INF/services/org.apache.ignite.plugin.PluginProvider new file mode 100644 index 0000000000000..d1a8786c00a3d --- /dev/null +++ b/modules/compatibility/src/test/resources/META-INF/services/org.apache.ignite.plugin.PluginProvider @@ -0,0 +1 @@ +org.apache.ignite.compatibility.testframework.plugins.TestCompatibilityPluginProvider diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java index 4d51853953f4d..05932d57eb1c8 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java @@ -40,10 +40,10 @@ import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteFileSystem; import org.apache.ignite.IgniteIllegalStateException; +import org.apache.ignite.IgniteLock; import org.apache.ignite.IgniteLogger; import org.apache.ignite.IgniteMessaging; import org.apache.ignite.IgniteQueue; -import org.apache.ignite.IgniteLock; import org.apache.ignite.IgniteScheduler; import org.apache.ignite.IgniteSemaphore; import org.apache.ignite.IgniteServices; @@ -137,27 +137,11 @@ public IgniteProcessProxy(IgniteConfiguration cfg, IgniteLogger log, Ignite locJ throws Exception { this.cfg = cfg; this.locJvmGrid = locJvmGrid; - this.log = log.getLogger("jvm-" + id.toString().substring(0, id.toString().indexOf('-'))); - - String cfgFileName = IgniteNodeRunner.storeToFile(cfg.setNodeId(id), resetDiscovery); - - Collection filteredJvmArgs = new ArrayList<>(); - - filteredJvmArgs.add("-ea"); + this.log = logger(log, "jvm-" + id.toString().substring(0, id.toString().indexOf('-'))); - Marshaller marsh = cfg.getMarshaller(); + String params = params(cfg, resetDiscovery); - if (marsh != null) - filteredJvmArgs.add("-D" + IgniteTestResources.MARSH_CLASS_NAME + "=" + marsh.getClass().getName()); - else - filteredJvmArgs.add("-D" + IgniteTestResources.MARSH_CLASS_NAME + "=" + BinaryMarshaller.class.getName()); - - for (String arg : U.jvmArgs()) { - if (arg.startsWith("-Xmx") || arg.startsWith("-Xms") || - arg.startsWith("-cp") || arg.startsWith("-classpath") || - (marsh != null && arg.startsWith("-D" + IgniteTestResources.MARSH_CLASS_NAME))) - filteredJvmArgs.add(arg); - } + Collection filteredJvmArgs = filteredJvmArgs(); final CountDownLatch rmtNodeStartedLatch = new CountDownLatch(1); @@ -165,8 +149,8 @@ public IgniteProcessProxy(IgniteConfiguration cfg, IgniteLogger log, Ignite locJ locJvmGrid.events().localListen(new NodeStartedListener(id, rmtNodeStartedLatch), EventType.EVT_NODE_JOINED); proc = GridJavaProcess.exec( - IgniteNodeRunner.class.getCanonicalName(), - cfgFileName, // Params. + igniteNodeRunnerClassName(), + params, this.log, // Optional closure to be called each time wrapped process prints line to system.out or system.err. new IgniteInClosure() { @@ -193,6 +177,68 @@ public IgniteProcessProxy(IgniteConfiguration cfg, IgniteLogger log, Ignite locJ } } + /** + * Creates new logger instance based on given logger and given category. + * + * @param log Base logger. + * @param ctgr Category. + * @return Initiated logger. + * @throws Exception In case of an error. + */ + protected IgniteLogger logger(IgniteLogger log, Object ctgr) throws Exception { + return log.getLogger(ctgr); + } + + /** + * Gets Ignite node runner class name. + * + * @return Node runner class name. + * @throws Exception In case of an error. + */ + protected String igniteNodeRunnerClassName() throws Exception { + return IgniteNodeRunner.class.getCanonicalName(); + } + + /** + * Creates parameters which will be passed to new Ignite Process as command line arguments. + * + * @param cfg Configuration. + * @param resetDiscovery Reset DiscoverySpi at the configuration. + * @return Params to be passed to new Ignite process. + * @throws Exception In case of an error. + */ + protected String params(IgniteConfiguration cfg, boolean resetDiscovery) throws Exception { + return IgniteNodeRunner.storeToFile(cfg.setNodeId(id), resetDiscovery); + } + + /** + * Creates list of JVM arguments to be used to start new Ignite process in separate JVM. + * + * @return JVM arguments. + * @throws Exception In case of an error. + */ + protected Collection filteredJvmArgs() throws Exception { + Collection filteredJvmArgs = new ArrayList<>(); + + filteredJvmArgs.add("-ea"); + + Marshaller marsh = cfg.getMarshaller(); + + if (marsh != null) + filteredJvmArgs.add("-D" + IgniteTestResources.MARSH_CLASS_NAME + "=" + marsh.getClass().getName()); + else + filteredJvmArgs.add("-D" + IgniteTestResources.MARSH_CLASS_NAME + "=" + BinaryMarshaller.class.getName()); + + for (String arg : U.jvmArgs()) { + if (arg.startsWith("-Xmx") || arg.startsWith("-Xms") || + arg.startsWith("-cp") || arg.startsWith("-classpath") || + (marsh != null && arg.startsWith("-D" + IgniteTestResources.MARSH_CLASS_NAME))) + filteredJvmArgs.add(arg); + } + + return filteredJvmArgs; + } + /** */ private static class NodeStartedListener extends IgnitePredicateX { diff --git a/pom.xml b/pom.xml index a456da47c0362..f386a1a06ab26 100644 --- a/pom.xml +++ b/pom.xml @@ -93,6 +93,7 @@ modules/kubernetes modules/zeromq modules/rocketmq + modules/compatibility @@ -101,6 +102,7 @@ examples modules/benchmarks + modules/compatibility modules/geospatial modules/hibernate-4.2 modules/hibernate-5.1 @@ -163,6 +165,14 @@ + + compatibility-tests + + modules/core + modules/compatibility + + + javadoc From 0a7ffcce6527111a68568da0a6d22d4a1eb306a4 Mon Sep 17 00:00:00 2001 From: Vyacheslav Daradur Date: Wed, 9 Aug 2017 17:09:08 +0300 Subject: [PATCH 2/7] IGNITE-5097 BinaryMarshaller should write ints in "varint" encoding where it makes sense (#21) --- .../apache/ignite/IgniteSystemProperties.java | 9 + .../internal/binary/BinaryEnumObjectImpl.java | 4 +- .../internal/binary/BinaryFieldImpl.java | 4 +- .../internal/binary/BinaryObjectImpl.java | 31 +- .../binary/BinaryObjectOffheapImpl.java | 18 +- .../BinarySerializedFieldComparator.java | 24 +- .../ignite/internal/binary/BinaryUtils.java | 294 ++++++++++++++++-- .../internal/binary/BinaryWriterExImpl.java | 136 ++++++-- .../binary/builder/BinaryBuilderReader.java | 78 +++-- .../builder/BinaryBuilderSerializer.java | 6 +- .../builder/BinaryEnumArrayLazyValue.java | 3 +- .../builder/BinaryObjectArrayLazyValue.java | 3 +- .../BinaryMarshallerIntArraysSizeTest.java | 41 +++ .../internal/binary/BinaryUtilsSelfTest.java | 88 ++++++ .../IgniteBinaryObjectsTestSuite.java | 4 + .../Binary/BinaryBuilderSelfTest.cs | 10 +- .../Binary/BinaryEqualityComparerTest.cs | 44 ++- .../Binary/BinarySelfTest.cs | 45 ++- .../Impl/Binary/BinaryObjectBuilder.cs | 22 +- .../Impl/Binary/BinaryUtils.cs | 166 +++++++--- 20 files changed, 857 insertions(+), 173 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerIntArraysSizeTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryUtilsSelfTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java index 2fa52b6f2f3df..0845c726fee28 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java @@ -702,6 +702,15 @@ public final class IgniteSystemProperties { */ public static final String IGNITE_ENABLE_FORCIBLE_NODE_KILL = "IGNITE_ENABLE_FORCIBLE_NODE_KILL"; + /** + * When set to {@code true}, Ignite switches to compatibility mode with versions that writing + * length of arrays in default format. + * + * Default is {@code false}, which means that length of arrays will be written in varint encoding. + * Varint encoding description. + */ + public static final String IGNITE_NO_VARINT_ARRAY_LENGTH = "IGNITE_NO_VARINT_ARRAY_LENGTH"; + /** * Enforces singleton. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java index 12a0fc352b99d..df8060f55c75b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java @@ -122,9 +122,9 @@ public BinaryEnumObjectImpl(BinaryContext ctx, byte[] arr) { if (this.typeId == GridBinaryMarshaller.UNREGISTERED_TYPE_ID) { assert arr[off] == GridBinaryMarshaller.STRING; - int len = BinaryPrimitives.readInt(arr, ++off); + int len = BinaryUtils.doReadArrayLength(arr, ++off); - off += 4; + off += BinaryUtils.sizeOfArrayLengthValue(len); byte[] bytes = BinaryPrimitives.readByteArray(arr, off, len); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldImpl.java index 59bd03dd39f0a..2cc0e96f21ede 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldImpl.java @@ -177,7 +177,7 @@ public int fieldId() { break; case GridBinaryMarshaller.STRING: { - int dataLen = buf.getInt(); + int dataLen = BinaryUtils.doReadArrayLength(buf); byte[] data = new byte[dataLen]; @@ -221,7 +221,7 @@ public int fieldId() { case GridBinaryMarshaller.DECIMAL: { int scale = buf.getInt(); - int dataLen = buf.getInt(); + int dataLen = BinaryUtils.doReadArrayLength(buf); byte[] data = new byte[dataLen]; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java index 1e706aedd1d86..bb1c4c1de6717 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java @@ -282,9 +282,11 @@ public void context(BinaryContext ctx) { assert arr[off] == GridBinaryMarshaller.STRING : arr[off]; - int len = BinaryPrimitives.readInt(arr, ++off); + int strLen = BinaryUtils.doReadArrayLength(arr, ++off); - String clsName = new String(arr, off + 4, len, UTF_8); + int len = BinaryUtils.sizeOfArrayLengthValue(strLen); + + String clsName = new String(arr, off + len, strLen, UTF_8); typeId = ctx.typeId(clsName); } @@ -334,9 +336,9 @@ public void context(BinaryContext ctx) { int typeId = BinaryPrimitives.readInt(arr, start + GridBinaryMarshaller.TYPE_ID_POS); if (typeId == GridBinaryMarshaller.UNREGISTERED_TYPE_ID) { - int len = BinaryPrimitives.readInt(arr, start + GridBinaryMarshaller.DFLT_HDR_LEN + 1); + int len = BinaryUtils.doReadArrayLength(arr, start + GridBinaryMarshaller.DFLT_HDR_LEN + 1); - return start + GridBinaryMarshaller.DFLT_HDR_LEN + len + 5; + return start + GridBinaryMarshaller.DFLT_HDR_LEN + 1 + BinaryUtils.sizeOfArrayLengthValue(len) + len; } else return start + GridBinaryMarshaller.DFLT_HDR_LEN; } @@ -423,9 +425,11 @@ else if (fieldOffLen == BinaryUtils.OFFSET_2) break; case GridBinaryMarshaller.STRING: { - int dataLen = BinaryPrimitives.readInt(arr, fieldPos + 1); + int dataLen = BinaryUtils.doReadArrayLength(arr, fieldPos + 1); + + int len = BinaryUtils.sizeOfArrayLengthValue(dataLen); - val = new String(arr, fieldPos + 5, dataLen, UTF_8); + val = new String(arr, fieldPos + 1 + len, dataLen, UTF_8); break; } @@ -470,9 +474,12 @@ else if (fieldOffLen == BinaryUtils.OFFSET_2) case GridBinaryMarshaller.DECIMAL: { int scale = BinaryPrimitives.readInt(arr, fieldPos + 1); + int len = 1 + 4; + + int dataLen = BinaryUtils.doReadArrayLength(arr, fieldPos + len); + len += BinaryUtils.sizeOfArrayLengthValue(dataLen); - int dataLen = BinaryPrimitives.readInt(arr, fieldPos + 5); - byte[] data = BinaryPrimitives.readByteArray(arr, fieldPos + 9, dataLen); + byte[] data = BinaryPrimitives.readByteArray(arr, fieldPos + len, dataLen); boolean negative = data[0] < 0; @@ -570,9 +577,9 @@ else if (fieldOffsetLen == BinaryUtils.OFFSET_2) break; case GridBinaryMarshaller.STRING: { - int dataLen = BinaryPrimitives.readInt(arr, fieldPos + 1); + int dataLen = BinaryUtils.doReadArrayLength(arr, fieldPos + 1); - totalLen = dataLen + 5; + totalLen = 1 + dataLen + BinaryUtils.sizeOfArrayLengthValue(dataLen); break; } @@ -588,9 +595,9 @@ else if (fieldOffsetLen == BinaryUtils.OFFSET_2) break; case GridBinaryMarshaller.DECIMAL: { - int dataLen = BinaryPrimitives.readInt(arr, fieldPos + 5); + int dataLen = BinaryUtils.doReadArrayLength(arr, fieldPos + 5); - totalLen = dataLen + 9; + totalLen = dataLen + 5 + BinaryUtils.sizeOfArrayLengthValue(dataLen); break; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java index 05f6963cabf71..daeca053421fd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java @@ -228,9 +228,9 @@ public BinaryObject heapCopy() { int typeId = BinaryPrimitives.readInt(ptr, start + GridBinaryMarshaller.TYPE_ID_POS); if (typeId == GridBinaryMarshaller.UNREGISTERED_TYPE_ID) { - int len = BinaryPrimitives.readInt(ptr, start + GridBinaryMarshaller.DFLT_HDR_LEN + 1); + int len = BinaryUtils.doReadArrayLength(ptr, start + GridBinaryMarshaller.DFLT_HDR_LEN + 1); - return start + GridBinaryMarshaller.DFLT_HDR_LEN + len + 5; + return start + GridBinaryMarshaller.DFLT_HDR_LEN + 1 + len + BinaryUtils.sizeOfArrayLengthValue(len); } else return start + GridBinaryMarshaller.DFLT_HDR_LEN; } @@ -317,8 +317,11 @@ else if (fieldOffLen == BinaryUtils.OFFSET_2) break; case GridBinaryMarshaller.STRING: { - int dataLen = BinaryPrimitives.readInt(ptr, fieldPos + 1); - byte[] data = BinaryPrimitives.readByteArray(ptr, fieldPos + 5, dataLen); + int dataLen = BinaryUtils.doReadArrayLength(ptr, fieldPos + 1); + + int len = BinaryUtils.sizeOfArrayLengthValue(dataLen); + + byte[] data = BinaryPrimitives.readByteArray(ptr, fieldPos + 1 + len, dataLen); val = new String(data, UTF_8); @@ -365,9 +368,12 @@ else if (fieldOffLen == BinaryUtils.OFFSET_2) case GridBinaryMarshaller.DECIMAL: { int scale = BinaryPrimitives.readInt(ptr, fieldPos + 1); + int len = 1 + 4; + + int dataLen = BinaryUtils.doReadArrayLength(ptr, fieldPos + len); + len += BinaryUtils.sizeOfArrayLengthValue(dataLen); - int dataLen = BinaryPrimitives.readInt(ptr, fieldPos + 5); - byte[] data = BinaryPrimitives.readByteArray(ptr, fieldPos + 9, dataLen); + byte[] data = BinaryPrimitives.readByteArray(ptr, fieldPos + len, dataLen); boolean negative = data[0] < 0; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinarySerializedFieldComparator.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinarySerializedFieldComparator.java index 43750732caca5..2db9bfdb05e90 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinarySerializedFieldComparator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinarySerializedFieldComparator.java @@ -180,6 +180,24 @@ private int readInt(int off) { return BinaryPrimitives.readInt(arr, curFieldPos + off); } + /** + * Reads value of length of an array, which can be presented in default format or varint encoding. + * Reading method depends whether this is offheap object and the constant {@link BinaryUtils#USE_VARINT_ARRAY_LENGTH}. + * Varint encoding description. + * + * If you need to know number of bytes which were used for storage of the read value, + * use the method {@link BinaryUtils#sizeOfArrayLengthValue(int)}. + * + * @param off Offset. + * @return Value of array's length. + */ + private int readArrayLength(int off) { + if (offheap()) + return BinaryUtils.doReadArrayLength(ptr, curFieldPos + off); + else + return BinaryUtils.doReadArrayLength(arr, curFieldPos + off); + } + /** * Read long value. * @@ -302,12 +320,12 @@ private static boolean isArray(@Nullable Object field) { */ private static boolean compareByteArrays(BinarySerializedFieldComparator c1, BinarySerializedFieldComparator c2, int off) { - int len = c1.readInt(off); + int len = c1.readArrayLength(off); - if (len != c2.readInt(off)) + if (len != c2.readArrayLength(off)) return false; else { - off += 4; + off += BinaryUtils.sizeOfArrayLengthValue(len); if (c1.offheap()) { if (c2.offheap()) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java index 74d17307ac1fa..a956e9b65cab5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java @@ -28,6 +28,7 @@ import java.lang.reflect.Proxy; import java.math.BigDecimal; import java.math.BigInteger; +import java.nio.ByteBuffer; import java.sql.Time; import java.sql.Timestamp; import java.util.ArrayList; @@ -58,6 +59,7 @@ import org.apache.ignite.binary.BinaryRawWriter; import org.apache.ignite.binary.BinaryType; import org.apache.ignite.binary.Binarylizable; +import org.apache.ignite.internal.binary.builder.BinaryBuilderReader; import org.apache.ignite.internal.binary.builder.BinaryLazyValue; import org.apache.ignite.internal.binary.streams.BinaryInputStream; import org.apache.ignite.internal.util.MutableSingletonList; @@ -139,6 +141,10 @@ public class BinaryUtils { public static final boolean FIELDS_SORTED_ORDER = IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_BINARY_SORT_OBJECT_FIELDS); + /** Whether to write arrays lengths in varint encoding. */ + public static final boolean USE_VARINT_ARRAY_LENGTH = + !IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_NO_VARINT_ARRAY_LENGTH); + /** Field type names. */ private static final String[] FIELD_TYPE_NAMES; @@ -1173,7 +1179,7 @@ public static boolean isEnum(Class cls) { * @return Value. */ public static byte[] doReadByteArray(BinaryInputStream in) { - int len = in.readInt(); + int len = doReadArrayLength(in); return in.readByteArray(len); } @@ -1182,7 +1188,7 @@ public static byte[] doReadByteArray(BinaryInputStream in) { * @return Value. */ public static boolean[] doReadBooleanArray(BinaryInputStream in) { - int len = in.readInt(); + int len = doReadArrayLength(in); return in.readBooleanArray(len); } @@ -1191,7 +1197,7 @@ public static boolean[] doReadBooleanArray(BinaryInputStream in) { * @return Value. */ public static short[] doReadShortArray(BinaryInputStream in) { - int len = in.readInt(); + int len = doReadArrayLength(in); return in.readShortArray(len); } @@ -1200,7 +1206,7 @@ public static short[] doReadShortArray(BinaryInputStream in) { * @return Value. */ public static char[] doReadCharArray(BinaryInputStream in) { - int len = in.readInt(); + int len = doReadArrayLength(in); return in.readCharArray(len); } @@ -1209,7 +1215,7 @@ public static char[] doReadCharArray(BinaryInputStream in) { * @return Value. */ public static int[] doReadIntArray(BinaryInputStream in) { - int len = in.readInt(); + int len = doReadArrayLength(in); return in.readIntArray(len); } @@ -1218,7 +1224,7 @@ public static int[] doReadIntArray(BinaryInputStream in) { * @return Value. */ public static long[] doReadLongArray(BinaryInputStream in) { - int len = in.readInt(); + int len = doReadArrayLength(in); return in.readLongArray(len); } @@ -1227,7 +1233,7 @@ public static long[] doReadLongArray(BinaryInputStream in) { * @return Value. */ public static float[] doReadFloatArray(BinaryInputStream in) { - int len = in.readInt(); + int len = doReadArrayLength(in); return in.readFloatArray(len); } @@ -1236,7 +1242,7 @@ public static float[] doReadFloatArray(BinaryInputStream in) { * @return Value. */ public static double[] doReadDoubleArray(BinaryInputStream in) { - int len = in.readInt(); + int len = doReadArrayLength(in); return in.readDoubleArray(len); } @@ -1246,7 +1252,8 @@ public static double[] doReadDoubleArray(BinaryInputStream in) { */ public static BigDecimal doReadDecimal(BinaryInputStream in) { int scale = in.readInt(); - byte[] mag = doReadByteArray(in); + int magLen = doReadArrayLength(in); + byte[] mag = in.readByteArray(magLen); boolean negative = mag[0] < 0; @@ -1266,7 +1273,8 @@ public static BigDecimal doReadDecimal(BinaryInputStream in) { */ public static String doReadString(BinaryInputStream in) { if (!in.hasArray()) { - byte[] arr = doReadByteArray(in); + int len = doReadArrayLength(in); + byte[] arr = in.readByteArray(len); if (USE_STR_SERIALIZATION_VER_2) return utf8BytesToStr(arr, 0, arr.length); @@ -1274,7 +1282,7 @@ public static String doReadString(BinaryInputStream in) { return new String(arr, UTF_8); } - int strLen = in.readInt(); + int strLen = doReadArrayLength(in); int pos = in.position(); @@ -1337,7 +1345,7 @@ public static Time doReadTime(BinaryInputStream in) { * @throws BinaryObjectException In case of error. */ public static BigDecimal[] doReadDecimalArray(BinaryInputStream in) throws BinaryObjectException { - int len = in.readInt(); + int len = doReadArrayLength(in); BigDecimal[] arr = new BigDecimal[len]; @@ -1362,7 +1370,7 @@ public static BigDecimal[] doReadDecimalArray(BinaryInputStream in) throws Binar * @throws BinaryObjectException In case of error. */ public static String[] doReadStringArray(BinaryInputStream in) throws BinaryObjectException { - int len = in.readInt(); + int len = doReadArrayLength(in); String[] arr = new String[len]; @@ -1387,7 +1395,7 @@ public static String[] doReadStringArray(BinaryInputStream in) throws BinaryObje * @throws BinaryObjectException In case of error. */ public static UUID[] doReadUuidArray(BinaryInputStream in) throws BinaryObjectException { - int len = in.readInt(); + int len = doReadArrayLength(in); UUID[] arr = new UUID[len]; @@ -1412,7 +1420,7 @@ public static UUID[] doReadUuidArray(BinaryInputStream in) throws BinaryObjectEx * @throws BinaryObjectException In case of error. */ public static Date[] doReadDateArray(BinaryInputStream in) throws BinaryObjectException { - int len = in.readInt(); + int len = doReadArrayLength(in); Date[] arr = new Date[len]; @@ -1437,7 +1445,7 @@ public static Date[] doReadDateArray(BinaryInputStream in) throws BinaryObjectEx * @throws BinaryObjectException In case of error. */ public static Timestamp[] doReadTimestampArray(BinaryInputStream in) throws BinaryObjectException { - int len = in.readInt(); + int len = doReadArrayLength(in); Timestamp[] arr = new Timestamp[len]; @@ -1462,7 +1470,7 @@ public static Timestamp[] doReadTimestampArray(BinaryInputStream in) throws Bina * @throws BinaryObjectException In case of error. */ public static Time[] doReadTimeArray(BinaryInputStream in) throws BinaryObjectException { - int len = in.readInt(); + int len = doReadArrayLength(in); Time[] arr = new Time[len]; @@ -1498,7 +1506,10 @@ public static BinaryObject doReadBinaryObject(BinaryInputStream in, BinaryContex return new BinaryObjectOffheapImpl(ctx, in.offheapPointer() + pos, start, len); } else { - byte[] arr = doReadByteArray(in); + int len = in.readInt(); + + byte[] arr = in.readByteArray(len); + int start = in.readInt(); BinaryObjectImpl binO = new BinaryObjectImpl(ctx, arr, start); @@ -1688,7 +1699,7 @@ private static BinaryEnumObjectImpl doReadBinaryEnum(BinaryInputStream in, Binar * @return Enum array. */ private static Object[] doReadBinaryEnumArray(BinaryInputStream in, BinaryContext ctx) { - int len = in.readInt(); + int len = doReadArrayLength(in); Object[] arr = (Object[])Array.newInstance(BinaryObject.class, len); @@ -1727,7 +1738,7 @@ public static Enum doReadEnum(BinaryInputStream in, Class cls) throws Bina */ public static Object[] doReadEnumArray(BinaryInputStream in, BinaryContext ctx, ClassLoader ldr, Class cls) throws BinaryObjectException { - int len = in.readInt(); + int len = doReadArrayLength(in); Object[] arr = (Object[])Array.newInstance(cls, len); @@ -1985,7 +1996,7 @@ public static Object[] doReadObjectArray(BinaryInputStream in, BinaryContext ctx Class compType = doReadClass(in, ctx, ldr, deserialize); - int len = in.readInt(); + int len = doReadArrayLength(in); Object[] arr = deserialize ? (Object[])Array.newInstance(compType, len) : new Object[len]; @@ -2359,7 +2370,7 @@ else if (c > 0x07FF) { } else { arr[position++] = (byte)(0xC0 | ((c >> 6) & 0x1F)); - arr[position++] = (byte)(0x80 | (c & 0x3F)); + arr[position++] = (byte)(0x80 | (c & 0x3F)); } } @@ -2485,6 +2496,245 @@ public static Map mergeEnumValues(String typeName, return mergedMap; } + /** + * Reads from {@link BinaryInputStream} value of length of an array, + * which can be presented in default format or varint encoding. + * Reading method depends on the constant {@link #USE_VARINT_ARRAY_LENGTH}. + * Varint encoding description. + * + * If you need to know number of bytes which were used for storage of the read value, + * use the method {@link #sizeOfArrayLengthValue(int)}. + * + * @param in BinaryInputStream. + * @return Length of an array. + */ + public static int doReadArrayLength(BinaryInputStream in) { + if (!USE_VARINT_ARRAY_LENGTH) + return in.readInt(); + + return doReadUnsignedVarint(in); + } + + /** + * Reads from {@link BinaryBuilderReader} value of length of an array, + * which can be presented in default format or varint encoding. + * Reading method depends on the constant {@link #USE_VARINT_ARRAY_LENGTH}. + * Varint encoding description. + * + * If you need to know number of bytes which were used for storage of the read value, + * use the method {@link #sizeOfArrayLengthValue(int)}. + * + * @param in BinaryBuilderReader. + * @return Length of an array. + */ + public static int doReadArrayLength(BinaryBuilderReader in) { + if (!USE_VARINT_ARRAY_LENGTH) + return in.readInt(); + + return doReadUnsignedVarint(in); + } + + /** + * Reads from {@link ByteBuffer} value of length of an array, + * which can be presented in default format or varint encoding. + * Reading method depends on the constant {@link #USE_VARINT_ARRAY_LENGTH}. + * Varint encoding description. + * + * If you need to know number of bytes which were used for storage of the read value, + * use the method {@link #sizeOfArrayLengthValue(int)}. + * + * @param buf ByteBuffer. + * @return Length of an array. + */ + public static int doReadArrayLength(ByteBuffer buf) { + if (!USE_VARINT_ARRAY_LENGTH) + return buf.getInt(); + + return doReadUnsignedVarint(buf); + } + + /** + * Reads value of length of an array, which can be presented in default format or varint encoding. + * Starts reading from given offset. + * Reading method depends on the constant {@link #USE_VARINT_ARRAY_LENGTH}. + * Varint encoding description. + * + * If you need to know number of bytes which were used for storage of the read value, + * use the method {@link #sizeOfArrayLengthValue(int)}. + * + * @param arr Bytes array. + * @param off Offset. + * @return Length of an array. + */ + public static int doReadArrayLength(byte[] arr, int off) { + if (!USE_VARINT_ARRAY_LENGTH) + return BinaryPrimitives.readInt(arr, off); + + return doReadUnsignedVarint(arr, off); + } + + /** + * Reads value of length of an array, which can be presented in default format or varint encoding. + * Starts reading from given offset. + * Reading method depends on the constant {@link #USE_VARINT_ARRAY_LENGTH}. + * Varint encoding description. + * + * If you need to know number of bytes which were used for storage of the read value, + * use the method {@link #sizeOfArrayLengthValue(int)}. + * + * @param ptr Pointer. + * @param off Offset. + * @return Length of an array. + */ + public static int doReadArrayLength(long ptr, int off) { + if (!USE_VARINT_ARRAY_LENGTH) + return BinaryPrimitives.readInt(ptr, off); + + return doReadUnsignedVarint(ptr, off); + } + + /** + * Returns the amount of bytes required to write length of an array, + * which can be presented in default format or varint encoding. + * Varint encoding description. + * It depends on the constant {@link #USE_VARINT_ARRAY_LENGTH}. + * + * @param len Array length. + * @return Amount of bytes. + */ + public static int sizeOfArrayLengthValue(int len) { + if (!USE_VARINT_ARRAY_LENGTH) + return 4; + + return sizeInUnsignedVarint(len); + } + + /** + * Reads from {@link BinaryInputStream} integer value which is presented in varint encoding. + * Varint encoding description. + * + * @param in BinaryInputStream. + * @return Decoded integer value. + * @throws BinaryObjectException If have been read more than 5 bytes. + */ + public static int doReadUnsignedVarint(BinaryInputStream in) throws BinaryObjectException { + int val = doReadUnsignedVarint(in.array(), in.position()); + + in.position(in.position() + sizeInUnsignedVarint(val)); + + return val; + } + + /** + * Reads from {@link BinaryBuilderReader} integer value which is presented in varint encoding. + * Varint encoding description. + * + * @param in BinaryBuilderReader. + * @return Decoded integer value. + * @throws BinaryObjectException If have been read more than 5 bytes. + */ + public static int doReadUnsignedVarint(BinaryBuilderReader in) throws BinaryObjectException { + int val = doReadUnsignedVarint(in.array(), in.position()); + + in.position(in.position() + sizeInUnsignedVarint(val)); + + return val; + } + + /** + * Reads from given {@link ByteBuffer} integer value which is presented in varint encoding. + * Starts reading from given offset. + * Varint encoding description. + * + * @param buf ByteBuffer. + * @return Decoded integer value. + * @throws BinaryObjectException If have been read more than 5 bytes. + */ + public static int doReadUnsignedVarint(ByteBuffer buf) throws BinaryObjectException { + int val = doReadUnsignedVarint(buf.array(), buf.position()); + + buf.position(buf.position() + sizeInUnsignedVarint(val)); + + return val; + } + + /** + * Reads from given bytes array integer value which is presented in varint encoding. + * Starts reading from given offset. + * Varint encoding description. + * + * @param arr Bytes array. + * @param off Offset. + * @return Decoded integer value. + * @throws BinaryObjectException If have been read more than 5 bytes. + */ + public static int doReadUnsignedVarint(byte[] arr, int off) throws BinaryObjectException { + int val = 0; + int bits = 0; + int b; + + while (((b = arr[off++]) & 0x80) != 0) { + val |= (b & 0x7F) << bits; + bits += 7; + + if (bits > 35) + throw new BinaryObjectException("Varint reading failed, sequence length is too long"); + } + + return val | (b << bits); + } + + /** + * Reads via given pointer integer value which is presented in varint encoding. + * Starts reading from given offset. + * + * @param ptr Pointer. + * @param off Offset. + * @return Decoded integer value. + * @throws BinaryObjectException If have been read more than 5 bytes. + */ + public static int doReadUnsignedVarint(long ptr, int off) throws BinaryObjectException { + int val = 0; + int bits = 0; + int b; + + while (((b = BinaryPrimitives.readByte(ptr, off++)) & 0x80) != 0) { + val |= (b & 0x7F) << bits; + bits += 7; + + if (bits > 35) + throw new BinaryObjectException("Varint reading failed, sequence length is too long"); + } + + return val | (b << bits); + } + + /** + * Returns the encoded size of the given unsigned integer value. + * Varint encoding description. + * + * @param val Value to be encoded. + * @return Encoded size. + */ + public static int sizeInUnsignedVarint(int val) { + if (val < 0) + return 5; + + if (val <= Byte.MAX_VALUE) + return 1; + + if (val <= 16383) + return 2; + + if (val <= 2097151) + return 3; + + if (val <= 268435455) + return 4; + + return 5; + } + /** * Enum type. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java index a7f645c1f4763..0afeb41c8ffe8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java @@ -382,7 +382,7 @@ public void doWriteDecimal(@Nullable BigDecimal val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4 + 4); + out.unsafeEnsure(1 + 4 + 5); out.unsafeWriteByte(GridBinaryMarshaller.DECIMAL); @@ -400,7 +400,7 @@ public void doWriteDecimal(@Nullable BigDecimal val) { if (negative) vals[0] |= -0x80; - out.unsafeWriteInt(vals.length); + doUnsafeWriteArrayLength(vals.length); out.writeByteArray(vals); } } @@ -419,10 +419,10 @@ public void doWriteString(@Nullable String val) { else strArr = val.getBytes(UTF_8); - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.STRING); - out.unsafeWriteInt(strArr.length); + doUnsafeWriteArrayLength(strArr.length); out.writeByteArray(strArr); } } @@ -504,9 +504,9 @@ void doWriteByteArray(@Nullable byte[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.BYTE_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); out.writeByteArray(val); } @@ -519,9 +519,9 @@ void doWriteShortArray(@Nullable short[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.SHORT_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); out.writeShortArray(val); } @@ -534,14 +534,82 @@ void doWriteIntArray(@Nullable int[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.INT_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); out.writeIntArray(val); } } + /** + * Writes value of length of an array, which can be written in default format or varint encoding. + * Writing method depends on the constant {@link BinaryUtils#USE_VARINT_ARRAY_LENGTH}. + * Varint encoding description. + * + * If you need to know necessary number of bytes for writing, + * use the method {@link BinaryUtils#sizeOfArrayLengthValue(int)}. + * + * @param val Value to write. + */ + public void doUnsafeWriteArrayLength(int val) { + if (!BinaryUtils.USE_VARINT_ARRAY_LENGTH) + out.unsafeWriteInt(val); + else + doUnsafeWriteUnsignedVarint(val); + } + + /** + * Writes value of length of an array, which can be written in default format or varint encoding. + * Writing method depends on the constant {@link BinaryUtils#USE_VARINT_ARRAY_LENGTH}. + * Varint encoding description. + * + * If you need to know necessary number of bytes for writing, + * use the method {@link BinaryUtils#sizeOfArrayLengthValue(int)}. + * + * @param val Value to write. + */ + public void doWriteArrayLength(int val) { + if (!BinaryUtils.USE_VARINT_ARRAY_LENGTH) + out.writeInt(val); + else + doWriteUnsignedVarint(val); + } + + /** + * Writes integer value in varint encoding. + * Varint encoding description. + * Value must be positive. + * + * @param val Value to write. + */ + public void doWriteUnsignedVarint(int val) { + while ((val & 0xFFFFFF80) != 0) { + out.writeByte((byte)((val & 0x7F) | 0x80)); + val >>>= 7; + } + + out.writeByte((byte)(val & 0x7F)); + } + + /** + * Writes integer value in varint encoding. + * Uses unsafe writing methods. + * Before calling, make sure that {@link #out} has 5 bytes for writing. + * Varint encoding description. + * Value must be positive. + * + * @param val Value to write. + */ + public void doUnsafeWriteUnsignedVarint(int val) { + while ((val & 0xFFFFFF80) != 0) { + out.unsafeWriteByte((byte)((val & 0x7F) | 0x80)); + val >>>= 7; + } + + out.unsafeWriteByte((byte)(val & 0x7F)); + } + /** * @param val Long array. */ @@ -549,9 +617,9 @@ void doWriteLongArray(@Nullable long[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.LONG_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); out.writeLongArray(val); } @@ -564,9 +632,9 @@ void doWriteFloatArray(@Nullable float[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.FLOAT_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); out.writeFloatArray(val); } @@ -579,9 +647,9 @@ void doWriteDoubleArray(@Nullable double[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.DOUBLE_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); out.writeDoubleArray(val); } @@ -594,9 +662,9 @@ void doWriteCharArray(@Nullable char[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.CHAR_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); out.writeCharArray(val); } @@ -609,9 +677,9 @@ void doWriteBooleanArray(@Nullable boolean[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.BOOLEAN_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); out.writeBooleanArray(val); } @@ -624,9 +692,9 @@ void doWriteDecimalArray(@Nullable BigDecimal[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.DECIMAL_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); for (BigDecimal str : val) doWriteDecimal(str); @@ -640,9 +708,9 @@ void doWriteStringArray(@Nullable String[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.STRING_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); for (String str : val) doWriteString(str); @@ -656,9 +724,9 @@ void doWriteUuidArray(@Nullable UUID[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.UUID_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); for (UUID uuid : val) doWriteUuid(uuid); @@ -672,9 +740,9 @@ void doWriteDateArray(@Nullable Date[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.DATE_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); for (Date date : val) doWriteDate(date); @@ -688,9 +756,9 @@ void doWriteTimestampArray(@Nullable Timestamp[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.TIMESTAMP_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); for (Timestamp ts : val) doWriteTimestamp(ts); @@ -704,9 +772,9 @@ void doWriteTimeArray(@Nullable Time[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.TIME_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); for (Time time : val) doWriteTime(time); @@ -737,7 +805,7 @@ void doWriteObjectArray(@Nullable Object[] val) throws BinaryObjectException { doWriteString(val.getClass().getComponentType().getName()); } - out.writeInt(val.length); + doWriteArrayLength(val.length); for (Object obj : val) doWriteObject(obj); @@ -862,7 +930,7 @@ void doWriteEnumArray(@Nullable Object[] val) { doWriteString(val.getClass().getComponentType().getName()); } - out.writeInt(val.length); + doWriteArrayLength(val.length); // TODO: Denis: Redundant data for each element of the array. for (Object o : val) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java index 2d10cf4490e06..ca7f4f874635b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java @@ -180,7 +180,7 @@ public byte readBytePositioned(int pos) { * @return Read length of array. */ public int readLength() { - return BinaryPrimitives.readInt(arr, pos); + return BinaryUtils.doReadArrayLength(arr, pos); } /** @@ -189,7 +189,7 @@ public int readLength() { * @return String length. */ public int readStringLength() { - return BinaryPrimitives.readInt(arr, pos); + return BinaryUtils.doReadArrayLength(arr, pos); } /** @@ -206,7 +206,9 @@ public String readString() { if (flag != GridBinaryMarshaller.STRING) throw new BinaryObjectException("Failed to deserialize String."); - int len = readInt(); + int len = readStringLength(); + + pos += BinaryUtils.sizeOfArrayLengthValue(len); String str = new String(arr, pos, len, UTF_8); @@ -264,17 +266,21 @@ public void skipValue() { case GridBinaryMarshaller.BYTE_ARR: case GridBinaryMarshaller.BOOLEAN_ARR: - len = 4 + readLength(); + len = readLength(); + len += BinaryUtils.sizeOfArrayLengthValue(len); break; case GridBinaryMarshaller.STRING: - len = 4 + readStringLength(); + len = readStringLength(); + len += BinaryUtils.sizeOfArrayLengthValue(len); break; case GridBinaryMarshaller.DECIMAL: - len = /** scale */ 4 + /** mag len */ 4 + /** mag bytes count */ readInt(4); + int magLen = BinaryUtils.doReadArrayLength(arr, pos); + + len = /* scale */ 4 + BinaryUtils.sizeOfArrayLengthValue(magLen) + magLen; break; @@ -300,19 +306,22 @@ public void skipValue() { case GridBinaryMarshaller.CHAR_ARR: case GridBinaryMarshaller.SHORT_ARR: - len = 4 + readLength() * 2; + len = readLength(); + len = len * 2 + BinaryUtils.sizeOfArrayLengthValue(len); break; case GridBinaryMarshaller.INT_ARR: case GridBinaryMarshaller.FLOAT_ARR: - len = 4 + readLength() * 4; + len = readLength(); + len = len * 4 + BinaryUtils.sizeOfArrayLengthValue(len); break; case GridBinaryMarshaller.LONG_ARR: case GridBinaryMarshaller.DOUBLE_ARR: - len = 4 + readLength() * 8; + len = readLength(); + len = len * 8 + BinaryUtils.sizeOfArrayLengthValue(len); break; @@ -324,7 +333,7 @@ public void skipValue() { case GridBinaryMarshaller.ENUM_ARR: case GridBinaryMarshaller.UUID_ARR: case GridBinaryMarshaller.STRING_ARR: { - int size = readInt(); + int size = BinaryUtils.doReadArrayLength(this); for (int i = 0; i < size; i++) skipValue(); @@ -580,12 +589,15 @@ public Object parseValue() { return arr[pos++] != 0; case GridBinaryMarshaller.DECIMAL: - plainLazyValLen = /** scale */ 4 + /** mag len */ 4 + /** mag bytes count */ readInt(4); + int magLen = BinaryUtils.doReadArrayLength(arr, pos + 4); + + plainLazyValLen = /* scale */ 4 + BinaryUtils.sizeOfArrayLengthValue(magLen) + magLen; break; case GridBinaryMarshaller.STRING: - plainLazyValLen = 4 + readStringLength(); + plainLazyValLen = readStringLength(); + plainLazyValLen += BinaryUtils.sizeOfArrayLengthValue(plainLazyValLen); break; @@ -610,49 +622,57 @@ public Object parseValue() { break; case GridBinaryMarshaller.BYTE_ARR: - plainLazyValLen = 4 + readLength(); + plainLazyValLen = readLength(); + plainLazyValLen += BinaryUtils.sizeOfArrayLengthValue(plainLazyValLen); modifiableLazyVal = true; break; case GridBinaryMarshaller.SHORT_ARR: - plainLazyValLen = 4 + readLength() * 2; + plainLazyValLen = readLength(); + plainLazyValLen = plainLazyValLen * 2 + BinaryUtils.sizeOfArrayLengthValue(plainLazyValLen); modifiableLazyVal = true; break; case GridBinaryMarshaller.INT_ARR: - plainLazyValLen = 4 + readLength() * 4; + plainLazyValLen = readLength(); + plainLazyValLen = plainLazyValLen * 4 + BinaryUtils.sizeOfArrayLengthValue(plainLazyValLen); modifiableLazyVal = true; break; case GridBinaryMarshaller.LONG_ARR: - plainLazyValLen = 4 + readLength() * 8; + plainLazyValLen = readLength(); + plainLazyValLen = plainLazyValLen * 8 + BinaryUtils.sizeOfArrayLengthValue(plainLazyValLen); modifiableLazyVal = true; break; case GridBinaryMarshaller.FLOAT_ARR: - plainLazyValLen = 4 + readLength() * 4; + plainLazyValLen = readLength(); + plainLazyValLen = plainLazyValLen * 4 + BinaryUtils.sizeOfArrayLengthValue(plainLazyValLen); modifiableLazyVal = true; break; case GridBinaryMarshaller.DOUBLE_ARR: - plainLazyValLen = 4 + readLength() * 8; + plainLazyValLen = readLength(); + plainLazyValLen = plainLazyValLen * 8 + BinaryUtils.sizeOfArrayLengthValue(plainLazyValLen); modifiableLazyVal = true; break; case GridBinaryMarshaller.CHAR_ARR: - plainLazyValLen = 4 + readLength() * 2; + plainLazyValLen = readLength(); + plainLazyValLen = plainLazyValLen * 2 + BinaryUtils.sizeOfArrayLengthValue(plainLazyValLen); modifiableLazyVal = true; break; case GridBinaryMarshaller.BOOLEAN_ARR: - plainLazyValLen = 4 + readLength(); + plainLazyValLen = readLength(); + plainLazyValLen += BinaryUtils.sizeOfArrayLengthValue(plainLazyValLen); modifiableLazyVal = true; break; @@ -661,7 +681,7 @@ public Object parseValue() { return new BinaryObjectArrayLazyValue(this); case GridBinaryMarshaller.DATE_ARR: { - int size = readInt(); + int size = BinaryUtils.doReadArrayLength(this); Date[] res = new Date[size]; @@ -684,7 +704,7 @@ public Object parseValue() { } case GridBinaryMarshaller.TIMESTAMP_ARR: { - int size = readInt(); + int size = BinaryUtils.doReadArrayLength(this); Timestamp[] res = new Timestamp[size]; @@ -716,7 +736,7 @@ public Object parseValue() { } case GridBinaryMarshaller.TIME_ARR: { - int size = readInt(); + int size = BinaryUtils.doReadArrayLength(this); Time[] res = new Time[size]; @@ -741,18 +761,22 @@ public Object parseValue() { case GridBinaryMarshaller.UUID_ARR: case GridBinaryMarshaller.STRING_ARR: case GridBinaryMarshaller.DECIMAL_ARR: { - int size = readInt(); + int size = BinaryUtils.doReadArrayLength(this); for (int i = 0; i < size; i++) { byte flag = arr[pos++]; if (flag == GridBinaryMarshaller.UUID) pos += 8 + 8; - else if (flag == GridBinaryMarshaller.STRING) - pos += 4 + readStringLength(); + else if (flag == GridBinaryMarshaller.STRING) { + int strLen = readStringLength(); + pos += BinaryUtils.sizeOfArrayLengthValue(strLen); + pos += strLen; + } else if (flag == GridBinaryMarshaller.DECIMAL) { pos += 4; // scale value - pos += 4 + readLength(); + int len = BinaryUtils.doReadArrayLength(arr, pos); + pos += BinaryUtils.sizeOfArrayLengthValue(len) + len; } else assert flag == GridBinaryMarshaller.NULL; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java index 018444c65123a..6d12ececb4740 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java @@ -191,7 +191,7 @@ public void writeValue(BinaryWriterExImpl writer, Object val, boolean forceCol, writer.writeByte(GridBinaryMarshaller.ENUM_ARR); writer.writeInt(compTypeId); - writer.writeInt(enumArr.length); + writer.doUnsafeWriteArrayLength(enumArr.length); for (Enum anEnum : enumArr) writeValue(writer, anEnum); @@ -216,7 +216,7 @@ public void writeValue(BinaryWriterExImpl writer, Object val, boolean forceCol, public void writeArray(BinaryWriterExImpl writer, byte elementType, Object[] arr, int compTypeId) { writer.writeByte(elementType); writer.writeInt(compTypeId); - writer.writeInt(arr.length); + writer.doUnsafeWriteArrayLength(arr.length); for (Object obj : arr) writeValue(writer, obj); @@ -232,7 +232,7 @@ public void writeArray(BinaryWriterExImpl writer, byte elementType, Object[] arr writer.writeByte(elementType); writer.writeInt(GridBinaryMarshaller.UNREGISTERED_TYPE_ID); writer.writeString(clsName); - writer.writeInt(arr.length); + writer.doUnsafeWriteArrayLength(arr.length); for (Object obj : arr) writeValue(writer, obj); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryEnumArrayLazyValue.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryEnumArrayLazyValue.java index 787ff638b995e..fe1bb88e6e617 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryEnumArrayLazyValue.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryEnumArrayLazyValue.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.binary.builder; +import org.apache.ignite.internal.binary.BinaryUtils; import org.apache.ignite.internal.binary.BinaryWriterExImpl; import org.apache.ignite.internal.binary.GridBinaryMarshaller; import org.apache.ignite.internal.util.typedef.internal.U; @@ -78,7 +79,7 @@ protected BinaryEnumArrayLazyValue(BinaryBuilderReader reader) { //skipping component type id reader.readInt(); - int size = reader.readInt(); + int size = BinaryUtils.doReadArrayLength(reader); BinaryBuilderEnum[] res = new BinaryBuilderEnum[size]; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectArrayLazyValue.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectArrayLazyValue.java index 8962107c77ac8..7ff2786195053 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectArrayLazyValue.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectArrayLazyValue.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.binary.builder; +import org.apache.ignite.internal.binary.BinaryUtils; import org.apache.ignite.internal.binary.GridBinaryMarshaller; import org.apache.ignite.internal.binary.BinaryWriterExImpl; import org.apache.ignite.internal.util.typedef.internal.U; @@ -62,7 +63,7 @@ protected BinaryObjectArrayLazyValue(BinaryBuilderReader reader) { clsName = null; } - int size = reader.readInt(); + int size = BinaryUtils.doReadArrayLength(reader); lazyValsArr = new Object[size]; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerIntArraysSizeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerIntArraysSizeTest.java new file mode 100644 index 0000000000000..fa6426db3aad5 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerIntArraysSizeTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.binary; + +import org.apache.ignite.IgniteSystemProperties; + +/** + * {@inheritDoc} + * + * Tests of Ignite compatibility mode with versions that writing length of arrays in default format. + */ +public class BinaryMarshallerIntArraysSizeTest extends BinaryMarshallerSelfTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + System.setProperty(IgniteSystemProperties.IGNITE_NO_VARINT_ARRAY_LENGTH, "true"); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + super.afterTestsStopped(); + + System.clearProperty(IgniteSystemProperties.IGNITE_NO_VARINT_ARRAY_LENGTH); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryUtilsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryUtilsSelfTest.java new file mode 100644 index 0000000000000..9bcacfbc667c9 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryUtilsSelfTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.binary; + +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.binary.streams.BinaryHeapInputStream; +import org.apache.ignite.logger.NullLogger; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * Tests for {@link BinaryUtils}. + */ +public class BinaryUtilsSelfTest extends GridCommonAbstractTest { + /** */ + private int[] ints; + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + ints = new int[] {Integer.MIN_VALUE, Short.MIN_VALUE, -90_000, Byte.MIN_VALUE, -5678, -234, -11, 0, 10, 200, 3000, Byte.MAX_VALUE, Short.MAX_VALUE, 40_000, 123_545, Integer.MAX_VALUE}; + } + + /** + * @throws Exception If failed. + */ + public void testUnsignedVarint() throws Exception { + BinaryWriterExImpl writer = createWriter(); + + int len = 0; + + for (int i = 0; i < ints.length; i++) { + writer.doWriteUnsignedVarint(ints[i]); + + assertEquals((len += BinaryUtils.sizeInUnsignedVarint(ints[i])), writer.array().length); + } + + BinaryHeapInputStream in = new BinaryHeapInputStream(writer.array()); + + for (int i = 0; i < ints.length; i++) + assertEquals(ints[i], BinaryUtils.doReadUnsignedVarint(in)); + } + + /** + * @throws Exception If failed. + */ + public void testSizeInUnsignedVarint() throws Exception { + assertEquals(1, BinaryUtils.sizeInUnsignedVarint(0)); + assertEquals(1, BinaryUtils.sizeInUnsignedVarint(Byte.MAX_VALUE)); + assertEquals(2, BinaryUtils.sizeInUnsignedVarint(Byte.MAX_VALUE + 1)); + + assertEquals(2, BinaryUtils.sizeInUnsignedVarint(128 * 128 - 1)); + assertEquals(3, BinaryUtils.sizeInUnsignedVarint(128 * 128)); + + assertEquals(3, BinaryUtils.sizeInUnsignedVarint(128 * 128 * 128 - 1)); + assertEquals(4, BinaryUtils.sizeInUnsignedVarint(128 * 128 * 128)); + + assertEquals(4, BinaryUtils.sizeInUnsignedVarint(128 * 128 * 128 * 128 - 1)); + assertEquals(5, BinaryUtils.sizeInUnsignedVarint(128 * 128 * 128 * 128)); + assertEquals(5, BinaryUtils.sizeInUnsignedVarint(Integer.MAX_VALUE)); + + // negative values + assertEquals(5, BinaryUtils.sizeInUnsignedVarint(-1)); + assertEquals(5, BinaryUtils.sizeInUnsignedVarint(Byte.MIN_VALUE)); + assertEquals(5, BinaryUtils.sizeInUnsignedVarint(Short.MIN_VALUE)); + assertEquals(5, BinaryUtils.sizeInUnsignedVarint(Integer.MIN_VALUE)); + } + + /** */ + private BinaryWriterExImpl createWriter() { + return new BinaryWriterExImpl(new BinaryContext(BinaryCachingMetadataHandler.create(), new IgniteConfiguration(), new NullLogger())); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java index c0211be4adbd7..92d5baeb8c5ea 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java @@ -28,6 +28,7 @@ import org.apache.ignite.internal.binary.BinaryFieldsOffheapSelfTest; import org.apache.ignite.internal.binary.BinaryFooterOffsetsHeapSelfTest; import org.apache.ignite.internal.binary.BinaryFooterOffsetsOffheapSelfTest; +import org.apache.ignite.internal.binary.BinaryMarshallerIntArraysSizeTest; import org.apache.ignite.internal.binary.BinaryMarshallerSelfTest; import org.apache.ignite.internal.binary.BinaryObjectBuilderAdditionalSelfTest; import org.apache.ignite.internal.binary.BinaryObjectBuilderDefaultMappersSelfTest; @@ -37,6 +38,7 @@ import org.apache.ignite.internal.binary.BinarySerialiedFieldComparatorSelfTest; import org.apache.ignite.internal.binary.BinarySimpleNameTestPropertySelfTest; import org.apache.ignite.internal.binary.BinaryTreeSelfTest; +import org.apache.ignite.internal.binary.BinaryUtilsSelfTest; import org.apache.ignite.internal.binary.GridBinaryAffinityKeySelfTest; import org.apache.ignite.internal.binary.GridBinaryMarshallerCtxDisabledSelfTest; import org.apache.ignite.internal.binary.GridBinaryWildcardsSelfTest; @@ -89,6 +91,8 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(BinaryTreeSelfTest.class); suite.addTestSuite(BinaryMarshallerSelfTest.class); + suite.addTestSuite(BinaryMarshallerIntArraysSizeTest.class); + suite.addTestSuite(BinaryUtilsSelfTest.class); suite.addTestSuite(BinaryObjectExceptionSelfTest.class); suite.addTestSuite(BinarySerialiedFieldComparatorSelfTest.class); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryBuilderSelfTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryBuilderSelfTest.cs index 61f90a3aa0535..cc70585275a6a 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryBuilderSelfTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryBuilderSelfTest.cs @@ -616,7 +616,15 @@ public void TestEquality() Assert.AreEqual(obj1, obj2); - Assert.AreEqual(-88648479, obj1.GetHashCode()); + if (BinaryUtils.UseVarintArrayLenght) + { + Assert.AreEqual(109523595, obj1.GetHashCode()); + } + else + { + Assert.AreEqual(-88648479, obj1.GetHashCode()); + } + Assert.AreEqual(obj1.GetHashCode(), obj2.GetHashCode()); Assert.AreEqual("myType [, int=1, str=foo]", Regex.Replace(obj1.ToString(), "idHash=\\d+", "")); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryEqualityComparerTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryEqualityComparerTest.cs index 4dc4d93923fd7..461a37fdfaa6d 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryEqualityComparerTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryEqualityComparerTest.cs @@ -102,22 +102,46 @@ public void TestBinaryObjects() Assert.IsFalse(BinaryArrayEqualityComparer.Equals(obj5, obj6)); // BinaryObject.GetHashCode. - Assert.AreEqual(1934949494, obj1.GetHashCode()); - Assert.AreEqual(-2013102781, obj2.GetHashCode()); - Assert.AreEqual(1424415317, obj3.GetHashCode()); - Assert.AreEqual(1771330338, obj4.GetHashCode()); + if (BinaryUtils.UseVarintArrayLenght) + { + Assert.AreEqual(860484896, obj1.GetHashCode()); + Assert.AreEqual(1207399917, obj2.GetHashCode()); + Assert.AreEqual(567081889, obj3.GetHashCode()); + Assert.AreEqual(913996910, obj4.GetHashCode()); + Assert.AreEqual(860514687, BinaryArrayEqualityComparer.GetHashCode(obj6)); + } + else + { + Assert.AreEqual(1934949494, obj1.GetHashCode()); + Assert.AreEqual(-2013102781, obj2.GetHashCode()); + Assert.AreEqual(1424415317, obj3.GetHashCode()); + Assert.AreEqual(1771330338, obj4.GetHashCode()); + Assert.AreEqual(1934979285, BinaryArrayEqualityComparer.GetHashCode(obj6)); + } + Assert.AreEqual(obj1.GetHashCode(), obj5.GetHashCode()); - Assert.AreEqual(1934979285, BinaryArrayEqualityComparer.GetHashCode(obj6)); // Comparer.GetHashCode. + if (BinaryUtils.UseVarintArrayLenght) + { + Assert.AreEqual(860484896, BinaryArrayEqualityComparer.GetHashCode(obj1)); + Assert.AreEqual(1207399917, BinaryArrayEqualityComparer.GetHashCode(obj2)); + Assert.AreEqual(567081889, BinaryArrayEqualityComparer.GetHashCode(obj3)); + Assert.AreEqual(913996910, BinaryArrayEqualityComparer.GetHashCode(obj4)); + Assert.AreEqual(860514687, BinaryArrayEqualityComparer.GetHashCode(obj6)); + } + else + { + Assert.AreEqual(1934949494, BinaryArrayEqualityComparer.GetHashCode(obj1)); + Assert.AreEqual(-2013102781, BinaryArrayEqualityComparer.GetHashCode(obj2)); + Assert.AreEqual(1424415317, BinaryArrayEqualityComparer.GetHashCode(obj3)); + Assert.AreEqual(1771330338, BinaryArrayEqualityComparer.GetHashCode(obj4)); + Assert.AreEqual(1934979285, BinaryArrayEqualityComparer.GetHashCode(obj6)); + } + Assert.AreEqual(2001751043, BinaryArrayEqualityComparer.GetHashCode(GetBinaryObject(0, null, 0))); Assert.AreEqual(194296580, BinaryArrayEqualityComparer.GetHashCode(GetBinaryObject(1, null, 0))); - Assert.AreEqual(1934949494, BinaryArrayEqualityComparer.GetHashCode(obj1)); - Assert.AreEqual(-2013102781, BinaryArrayEqualityComparer.GetHashCode(obj2)); - Assert.AreEqual(1424415317, BinaryArrayEqualityComparer.GetHashCode(obj3)); - Assert.AreEqual(1771330338, BinaryArrayEqualityComparer.GetHashCode(obj4)); Assert.AreEqual(BinaryArrayEqualityComparer.GetHashCode(obj1), BinaryArrayEqualityComparer.GetHashCode(obj5)); - Assert.AreEqual(1934979285, BinaryArrayEqualityComparer.GetHashCode(obj6)); // GetHashCode consistency. foreach (var obj in new[] {obj1, obj2, obj3, obj4, obj5, obj6}) diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinarySelfTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinarySelfTest.cs index e24dca0ddb5cc..9083707432678 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinarySelfTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinarySelfTest.cs @@ -627,7 +627,26 @@ public void TestWriteEnumArrayRegistered() Assert.AreEqual(vals, newVals); } - + + /** + * Checks the writing an integer value in varint encoding. + */ + [Test] + public void TestWriteUvarint() + { + int[] vals = {int.MinValue, short.MinValue, -16384, -128, 0, 127, 16383, short.MaxValue, int.MaxValue}; + + var stream = new BinaryHeapStream(64); + + foreach (int val in vals) + BinaryUtils.WriteUvarint(val, stream); + + stream.Seek(0, SeekOrigin.Begin); + + foreach (int val in vals) + Assert.AreEqual(val, BinaryUtils.ReadUvarint(stream)); + } + /// /// Test object with dates. /// @@ -2619,5 +2638,29 @@ public override int GetHashCode() return !left.Equals(right); } } + + /** + * Tests Ignite compatibility mode with versions that writing length of arrays in default format. + */ + [Test] + public void TestNoVarintArrayLengthMode() + { + if (Environment.GetEnvironmentVariable(BinaryUtils.IgniteNoVarintArrayLength) != "true") + { + // Run "TestNoVarintArrayLengthMode" in a separate process with changed setting. + Environment.SetEnvironmentVariable(BinaryUtils.IgniteNoVarintArrayLength, "true"); + + TestUtils.RunTestInNewProcess(GetType().FullName, "TestNoVarintArrayLengthMode"); + } + } + + /** + * Test tear down. + */ + [TearDown] + public void TearDown() + { + Environment.SetEnvironmentVariable(BinaryUtils.IgniteNoVarintArrayLength, null); + } } } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectBuilder.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectBuilder.cs index c310b3afa4064..3d8de5dac2cea 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectBuilder.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectBuilder.cs @@ -869,9 +869,9 @@ internal void ProcessBuilder(IBinaryStream outStream, BinaryObjectBuilder builde case BinaryUtils.TypeDecimal: TransferBytes(inStream, outStream, 4); // Transfer scale - int magLen = inStream.ReadInt(); // Transfer magnitude length. + int magLen = BinaryUtils.ReadArrayLength(inStream); // Transfer magnitude length. - outStream.WriteInt(magLen); + BinaryUtils.WriteArrayLength(magLen, outStream); TransferBytes(inStream, outStream, magLen); // Transfer magnitude. @@ -936,9 +936,9 @@ internal void ProcessBuilder(IBinaryStream outStream, BinaryObjectBuilder builde case BinaryUtils.TypeArrayString: case BinaryUtils.TypeArrayGuid: case BinaryUtils.TypeArrayTimestamp: - int arrLen = inStream.ReadInt(); + int arrLen = BinaryUtils.ReadArrayLength(inStream); - outStream.WriteInt(arrLen); + BinaryUtils.WriteArrayLength(arrLen, outStream); for (int i = 0; i < arrLen; i++) Mutate0(ctx, inStream, outStream, false, null); @@ -958,9 +958,9 @@ internal void ProcessBuilder(IBinaryStream outStream, BinaryObjectBuilder builde BinaryUtils.WriteString(BinaryUtils.ReadString(inStream), outStream); // String data. } - arrLen = inStream.ReadInt(); + arrLen = BinaryUtils.ReadArrayLength(inStream); - outStream.WriteInt(arrLen); + BinaryUtils.WriteArrayLength(arrLen, outStream); for (int i = 0; i < arrLen; i++) Mutate0(ctx, inStream, outStream, false, EmptyVals); @@ -995,7 +995,11 @@ internal void ProcessBuilder(IBinaryStream outStream, BinaryObjectBuilder builde break; case BinaryUtils.TypeBinary: - TransferArray(inStream, outStream, 1); // Data array. + int len = inStream.ReadInt(); + + outStream.WriteInt(len); + + TransferBytes(inStream, outStream, len); // Data array. TransferBytes(inStream, outStream, 4); // Offset in array. break; @@ -1034,9 +1038,9 @@ private static void TransferBytes(BinaryHeapStream inStream, IBinaryStream outSt private static void TransferArray(BinaryHeapStream inStream, IBinaryStream outStream, int elemSize) { - int len = inStream.ReadInt(); + int len = BinaryUtils.ReadArrayLength(inStream); - outStream.WriteInt(len); + BinaryUtils.WriteArrayLength(len, outStream); TransferBytes(inStream, outStream, elemSize * len); } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs index 91a536edc7438..98a2b8383dde6 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs @@ -252,6 +252,13 @@ internal static class BinaryUtils public static readonly bool UseStringSerializationVer2 = (Environment.GetEnvironmentVariable(IgniteBinaryMarshallerUseStringSerializationVer2) ?? "false") == "true"; + /** Length of arrays format environment variable. */ + public const string IgniteNoVarintArrayLength = "IGNITE_NO_VARINT_ARRAY_LENGTH"; + + /** Length of arrays writing mode. */ + public static readonly bool UseVarintArrayLenght = + (Environment.GetEnvironmentVariable(IgniteNoVarintArrayLength) ?? "false") != "true"; + /// /// Default marshaller. /// @@ -267,7 +274,7 @@ public static Marshaller Marshaller */ public static void WriteBooleanArray(bool[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); stream.WriteBoolArray(vals); } @@ -279,7 +286,7 @@ public static void WriteBooleanArray(bool[] vals, IBinaryStream stream) */ public static bool[] ReadBooleanArray(IBinaryStream stream) { - int len = stream.ReadInt(); + int len = ReadArrayLength(stream); return stream.ReadBoolArray(len); } @@ -292,7 +299,7 @@ public static bool[] ReadBooleanArray(IBinaryStream stream) */ public static void WriteByteArray(byte[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); stream.WriteByteArray(vals); } @@ -304,7 +311,9 @@ public static void WriteByteArray(byte[] vals, IBinaryStream stream) */ public static byte[] ReadByteArray(IBinaryStream stream) { - return stream.ReadByteArray(stream.ReadInt()); + int len = ReadArrayLength(stream); + + return stream.ReadByteArray(len); } /** @@ -314,7 +323,7 @@ public static byte[] ReadByteArray(IBinaryStream stream) */ public static unsafe sbyte[] ReadSbyteArray(IBinaryStream stream) { - int len = stream.ReadInt(); + int len = ReadArrayLength(stream); sbyte[] res = new sbyte[len]; @@ -333,7 +342,7 @@ public static unsafe sbyte[] ReadSbyteArray(IBinaryStream stream) */ public static void WriteShortArray(short[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); stream.WriteShortArray(vals); } @@ -345,7 +354,7 @@ public static void WriteShortArray(short[] vals, IBinaryStream stream) */ public static unsafe ushort[] ReadUshortArray(IBinaryStream stream) { - int len = stream.ReadInt(); + int len = ReadArrayLength(stream); ushort[] res = new ushort[len]; @@ -364,7 +373,9 @@ public static unsafe ushort[] ReadUshortArray(IBinaryStream stream) */ public static short[] ReadShortArray(IBinaryStream stream) { - return stream.ReadShortArray(stream.ReadInt()); + int len = ReadArrayLength(stream); + + return stream.ReadShortArray(len); } /** @@ -374,7 +385,7 @@ public static short[] ReadShortArray(IBinaryStream stream) */ public static void WriteIntArray(int[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); stream.WriteIntArray(vals); } @@ -386,7 +397,9 @@ public static void WriteIntArray(int[] vals, IBinaryStream stream) */ public static int[] ReadIntArray(IBinaryStream stream) { - return stream.ReadIntArray(stream.ReadInt()); + int len = ReadArrayLength(stream); + + return stream.ReadIntArray(len); } /** @@ -396,7 +409,7 @@ public static int[] ReadIntArray(IBinaryStream stream) */ public static unsafe uint[] ReadUintArray(IBinaryStream stream) { - int len = stream.ReadInt(); + int len = ReadArrayLength(stream); uint[] res = new uint[len]; @@ -415,7 +428,7 @@ public static unsafe uint[] ReadUintArray(IBinaryStream stream) */ public static void WriteLongArray(long[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); stream.WriteLongArray(vals); } @@ -427,7 +440,9 @@ public static void WriteLongArray(long[] vals, IBinaryStream stream) */ public static long[] ReadLongArray(IBinaryStream stream) { - return stream.ReadLongArray(stream.ReadInt()); + int len = ReadArrayLength(stream); + + return stream.ReadLongArray(len); } /** @@ -437,7 +452,7 @@ public static long[] ReadLongArray(IBinaryStream stream) */ public static unsafe ulong[] ReadUlongArray(IBinaryStream stream) { - int len = stream.ReadInt(); + int len = ReadArrayLength(stream); ulong[] res = new ulong[len]; @@ -456,7 +471,7 @@ public static unsafe ulong[] ReadUlongArray(IBinaryStream stream) */ public static void WriteCharArray(char[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); stream.WriteCharArray(vals); } @@ -468,7 +483,7 @@ public static void WriteCharArray(char[] vals, IBinaryStream stream) */ public static char[] ReadCharArray(IBinaryStream stream) { - int len = stream.ReadInt(); + int len = ReadArrayLength(stream); return stream.ReadCharArray(len); } @@ -480,7 +495,7 @@ public static char[] ReadCharArray(IBinaryStream stream) */ public static void WriteFloatArray(float[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); stream.WriteFloatArray(vals); } @@ -492,7 +507,7 @@ public static void WriteFloatArray(float[] vals, IBinaryStream stream) */ public static float[] ReadFloatArray(IBinaryStream stream) { - int len = stream.ReadInt(); + int len = ReadArrayLength(stream); return stream.ReadFloatArray(len); } @@ -504,7 +519,7 @@ public static float[] ReadFloatArray(IBinaryStream stream) */ public static void WriteDoubleArray(double[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); stream.WriteDoubleArray(vals); } @@ -516,7 +531,7 @@ public static void WriteDoubleArray(double[] vals, IBinaryStream stream) */ public static double[] ReadDoubleArray(IBinaryStream stream) { - int len = stream.ReadInt(); + int len = ReadArrayLength(stream); return stream.ReadDoubleArray(len); } @@ -557,7 +572,7 @@ public static void WriteTimestamp(DateTime val, IBinaryStream stream) /// Stream. public static void WriteTimestampArray(DateTime?[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); foreach (DateTime? val in vals) { @@ -585,7 +600,7 @@ public static unsafe void WriteString(string val, IBinaryStream stream) { int byteCnt = GetUtf8ByteCount(chars, charCnt); - stream.WriteInt(byteCnt); + WriteArrayLength(byteCnt, stream); stream.WriteString(chars, charCnt, byteCnt, Utf8); } @@ -775,7 +790,7 @@ public static string ReadString(IBinaryStream stream) */ public static void WriteStringArray(string[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); foreach (string val in vals) { @@ -789,6 +804,78 @@ public static void WriteStringArray(string[] vals, IBinaryStream stream) } } + /** + * + * Writes value of length of an array, which can be written in default format or varint encoding. + * Writing method depends on the constant . + * + * Varint encoding description. + * Integer value. + * Stream. + */ + public static void WriteArrayLength(int val, IBinaryStream stream) + { + if (!UseVarintArrayLenght) + stream.WriteInt(val); + else + WriteUvarint(val, stream); + } + + /** + * + * Reads value of length of an array, which can be presented in default format or varint encoding. + * Reading method depends on the constant . + * + * Varint encoding description. + * Stream. + * Integer value. + */ + public static int ReadArrayLength(IBinaryStream stream) + { + return !UseVarintArrayLenght ? stream.ReadInt() : ReadUvarint(stream); + } + + /** + * Writes an integer value in varint encoding. + * Varint encoding description. + * Integer value. + * Stream. + */ + public static void WriteUvarint(int val, IBinaryStream stream) + { + while ((val & 0xFFFFFF80) != 0) + { + stream.WriteByte((byte)((val & 0x7F) | 0x80)); + val = (int)((uint)val >> 7); + } + + stream.WriteByte((byte)(val & 0x7F)); + } + + /** + * Reads an integer value which is presented in varint encoding. + * Varint encoding description. + * Stream. + * Decoded integer value. + */ + public static int ReadUvarint(IBinaryStream stream) + { + int val = 0; + int bits = 0; + int b; + + while (((b = stream.ReadByte()) & 0x80) != 0) + { + val |= (int)((uint)(b & 0x7F) << bits); + bits += 7; + + if (bits > 35) + throw new BinaryObjectException("Varint reading failed, sequence length is too long"); + } + + return val | (int)((uint)b << bits); + } + /** * Write decimal value. * Decimal value. @@ -816,7 +903,7 @@ public static void WriteDecimal(decimal val, IBinaryStream stream) if (idx == -1) { // Writing zero. - stream.WriteInt(1); + WriteArrayLength(1, stream); stream.WriteByte(0); } else @@ -840,14 +927,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream) { if ((part24 & 0x80) == 0x80) { - stream.WriteInt(len + 1); + WriteArrayLength(len + 1, stream); stream.WriteByte((byte)(neg ? -0x80 : ByteZero)); neg = false; } else - stream.WriteInt(len); + WriteArrayLength(len, stream); stream.WriteByte((byte)(neg ? ((sbyte)part24 | -0x80) : part24)); stream.WriteByte((byte)part16); @@ -858,14 +945,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream) { if ((part16 & 0x80) == 0x80) { - stream.WriteInt(len); + WriteArrayLength(len, stream); stream.WriteByte((byte)(neg ? -0x80 : ByteZero)); neg = false; } else - stream.WriteInt(len - 1); + WriteArrayLength(len - 1, stream); stream.WriteByte((byte)(neg ? ((sbyte)part16 | -0x80) : part16)); stream.WriteByte((byte)part8); @@ -875,14 +962,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream) { if ((part8 & 0x80) == 0x80) { - stream.WriteInt(len - 1); + WriteArrayLength(len - 1, stream); stream.WriteByte((byte)(neg ? -0x80 : ByteZero)); neg = false; } else - stream.WriteInt(len - 2); + WriteArrayLength(len - 2, stream); stream.WriteByte((byte)(neg ? ((sbyte)part8 | -0x80) : part8)); stream.WriteByte((byte)part0); @@ -891,14 +978,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream) { if ((part0 & 0x80) == 0x80) { - stream.WriteInt(len - 2); + WriteArrayLength(len - 2, stream); stream.WriteByte((byte)(neg ? -0x80 : ByteZero)); neg = false; } else - stream.WriteInt(len - 3); + WriteArrayLength(len - 3, stream); stream.WriteByte((byte)(neg ? ((sbyte)part0 | -0x80) : part0)); } @@ -979,7 +1066,7 @@ public static void WriteDecimal(decimal val, IBinaryStream stream) */ public static void WriteDecimalArray(decimal?[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); foreach (var val in vals) { @@ -1001,7 +1088,7 @@ public static void WriteDecimalArray(decimal?[] vals, IBinaryStream stream) */ public static decimal?[] ReadDecimalArray(IBinaryStream stream) { - int len = stream.ReadInt(); + int len = ReadArrayLength(stream); var vals = new decimal?[len]; @@ -1141,7 +1228,7 @@ public static unsafe Guid ReadGuidSlow(IBinaryStream stream) /// Stream. public static void WriteGuidArray(Guid?[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); foreach (Guid? val in vals) { @@ -1187,7 +1274,7 @@ public static void WriteArray(Array val, BinaryWriter ctx, int? elemTypeId = nul ctx.WriteString(elemType.FullName); } - stream.WriteInt(val.Length); + WriteArrayLength(val.Length, stream); for (int i = 0; i < val.Length; i++) ctx.Write(val.GetValue(i)); @@ -1233,7 +1320,7 @@ public static T[] ReadArray(BinaryReader ctx, bool typed) ctx.ReadString(); } - int len = stream.ReadInt(); + int len = ReadArrayLength(stream); var vals = new T[len]; @@ -1252,7 +1339,7 @@ public static T[] ReadArray(BinaryReader ctx, bool typed) /// Timestamp array. public static DateTime?[] ReadTimestampArray(IBinaryStream stream) { - int len = stream.ReadInt(); + int len = ReadArrayLength(stream); DateTime?[] vals = new DateTime?[len]; @@ -1428,7 +1515,8 @@ public static IDictionary ReadDictionary(BinaryReader ctx, Func Date: Wed, 9 Aug 2017 20:08:50 +0300 Subject: [PATCH 3/7] ignite-5736: added VarintArraysSizeCompatibilityTest --- .../VarintArraysSizeCompatibilityTest.java | 181 ++++++++++++++++++ .../IgniteCompatibilityBasicTestSuite.java | 3 + 2 files changed, 184 insertions(+) create mode 100644 modules/compatibility/src/test/java/org/apache/ignite/compatibility/binary/VarintArraysSizeCompatibilityTest.java diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/binary/VarintArraysSizeCompatibilityTest.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/binary/VarintArraysSizeCompatibilityTest.java new file mode 100644 index 0000000000000..9dae652648db7 --- /dev/null +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/binary/VarintArraysSizeCompatibilityTest.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compatibility.binary; + +import java.math.BigDecimal; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.Arrays; +import java.util.Date; +import java.util.UUID; +import javax.cache.processor.EntryProcessor; +import javax.cache.processor.EntryProcessorException; +import javax.cache.processor.MutableEntry; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.CacheRebalanceMode; +import org.apache.ignite.compatibility.testframework.junits.IgniteCompatibilityAbstractTest; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.processors.cache.GridCacheAbstractFullApiSelfTest; +import org.apache.ignite.lang.IgniteInClosure; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.testframework.GridTestUtils; + +/** + * Tests of storing data in compatibility mode. + */ +public class VarintArraysSizeCompatibilityTest extends IgniteCompatibilityAbstractTest { + @Override protected IgniteConfiguration getConfiguration() throws Exception { + IgniteConfiguration cfg = super.getConfiguration(); + cfg.setPeerClassLoadingEnabled(true); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + System.setProperty(IgniteSystemProperties.IGNITE_NO_VARINT_ARRAY_LENGTH, "true"); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + super.afterTestsStopped(); + + System.clearProperty(IgniteSystemProperties.IGNITE_NO_VARINT_ARRAY_LENGTH); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(); + } + + /** {@inheritDoc} */ + @Override protected boolean isMultiJvm() { + return true; + } + + /** + * @throws Exception If failed. + */ + public void testArraysStoringInCompatibilityMode() throws Exception { + startGrid(1, "2.1.0", new PostConfigurationClosure()); + + startGrid(2, "2.1.0", new PostConfigurationClosure()); + + Ignite ignite = startGrid(0); + + CacheConfiguration cacheCfg = defaultCacheConfiguration(); + cacheCfg.setCacheMode(CacheMode.REPLICATED); + cacheCfg.setName("varintTestCache"); + cacheCfg.setRebalanceMode(CacheRebalanceMode.SYNC); + + IgniteCache cache = ignite.createCache(cacheCfg); + + final TestObject obj = new TestObject(); + + final String key = "key"; + + cache.put(key, obj); + + cache.invoke( + key, + new EntryProcessor() { + @Override public TestObject process(MutableEntry entry, + Object... objects) throws EntryProcessorException { + TestObject val = entry.getValue(); + + assertEquals(obj, val); + + return val; + } + }); + } + + /** */ + private static class TestObject { + byte[] bArr = new byte[] {1, 2, 3}; + boolean[] boolArr = new boolean[] {true, false, true}; + char[] cArr = new char[] {1, 2, 3}; + short[] sArr = new short[] {1, 2, 3}; + int[] iArr = new int[] {1, 2, 3}; + long[] lArr = new long[] {1, 2, 3}; + float[] fArr = new float[] {1.1f, 2.2f, 3.3f}; + double[] dArr = new double[] {1.1d, 2.2d, 3.3d}; + BigDecimal[] bdArr = new BigDecimal[] {BigDecimal.ZERO, new BigDecimal(1000), new BigDecimal(123456789)}; + String[] strArr = new String[] {"str1", "str2", "str3"}; + UUID[] uuidArr = new UUID[] {UUID.randomUUID(), UUID.randomUUID(), UUID.randomUUID()}; + Date[] dateArr = new Date[] {new Date(11111), new Date(22222), new Date(33333)}; + Timestamp[] tsArr = new Timestamp[] {new Timestamp(11111), new Timestamp(22222), new Timestamp(33333)}; + Time[] timeArr = new Time[] {new Time(11111), new Time(22222), new Time(33333)}; + TestEnum[] enumArr = new TestEnum[] {TestEnum.A, TestEnum.B, TestEnum.C}; + Object[] objArr = new Object[] {UUID.randomUUID(), UUID.randomUUID(), UUID.randomUUID()}; + + /** {@inheritDoc} */ + @Override public int hashCode() { + int result = Arrays.hashCode(bArr); + result = 31 * result + Arrays.hashCode(boolArr); + result = 31 * result + Arrays.hashCode(cArr); + result = 31 * result + Arrays.hashCode(sArr); + result = 31 * result + Arrays.hashCode(iArr); + result = 31 * result + Arrays.hashCode(lArr); + result = 31 * result + Arrays.hashCode(fArr); + result = 31 * result + Arrays.hashCode(dArr); + result = 31 * result + Arrays.hashCode(bdArr); + result = 31 * result + Arrays.hashCode(strArr); + result = 31 * result + Arrays.hashCode(uuidArr); + result = 31 * result + Arrays.hashCode(dateArr); + result = 31 * result + Arrays.hashCode(tsArr); + result = 31 * result + Arrays.hashCode(timeArr); + result = 31 * result + Arrays.hashCode(enumArr); + result = 31 * result + Arrays.hashCode(objArr); + return result; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object obj) { + return GridTestUtils.deepEquals(this, obj); + } + } + + /** */ + private enum TestEnum { + A, B, C + } + + /** */ + private static class PostConfigurationClosure implements IgniteInClosure { + /** {@inheritDoc} */ + @Override public void apply(IgniteConfiguration cfg) { + cfg.setLocalHost("127.0.0.1"); + + TcpDiscoverySpi disco = new TcpDiscoverySpi(); + disco.setIpFinder(GridCacheAbstractFullApiSelfTest.LOCAL_IP_FINDER); + + cfg.setDiscoverySpi(disco); + + cfg.setPeerClassLoadingEnabled(true); + } + } +} \ No newline at end of file diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java index 78e769a5ad80f..307d8c8ff2412 100644 --- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java @@ -18,6 +18,7 @@ package org.apache.ignite.compatibility.testsuites; import junit.framework.TestSuite; +import org.apache.ignite.compatibility.binary.VarintArraysSizeCompatibilityTest; import org.apache.ignite.compatibility.spi.MultiVersionClusterTest; /** @@ -32,6 +33,8 @@ public static TestSuite suite() throws Exception { TestSuite suite = new TestSuite("Ignite Compatibility Basic Test Suite"); suite.addTestSuite(MultiVersionClusterTest.class); + + suite.addTestSuite(VarintArraysSizeCompatibilityTest.class); return suite; } From 1867d9cbe8886996e36eb50d5752ee5e07e68494 Mon Sep 17 00:00:00 2001 From: Vyacheslav Daradur Date: Thu, 10 Aug 2017 13:39:07 +0300 Subject: [PATCH 4/7] ignite-5736: added missed javadoc --- .../compatibility/binary/VarintArraysSizeCompatibilityTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/binary/VarintArraysSizeCompatibilityTest.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/binary/VarintArraysSizeCompatibilityTest.java index 9dae652648db7..e08e52efe9072 100644 --- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/binary/VarintArraysSizeCompatibilityTest.java +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/binary/VarintArraysSizeCompatibilityTest.java @@ -43,6 +43,7 @@ * Tests of storing data in compatibility mode. */ public class VarintArraysSizeCompatibilityTest extends IgniteCompatibilityAbstractTest { + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration() throws Exception { IgniteConfiguration cfg = super.getConfiguration(); cfg.setPeerClassLoadingEnabled(true); From f01f2e39f4452312f8f8d36dbf4d0f6ddcf32e1c Mon Sep 17 00:00:00 2001 From: Vyacheslav Daradur Date: Thu, 10 Aug 2017 18:45:02 +0300 Subject: [PATCH 5/7] ignite-5736: review fixes --- .../VarintArraysSizeCompatibilityTest.java | 60 +++++++------------ 1 file changed, 20 insertions(+), 40 deletions(-) diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/binary/VarintArraysSizeCompatibilityTest.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/binary/VarintArraysSizeCompatibilityTest.java index e08e52efe9072..614b70b1bfadd 100644 --- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/binary/VarintArraysSizeCompatibilityTest.java +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/binary/VarintArraysSizeCompatibilityTest.java @@ -20,7 +20,6 @@ import java.math.BigDecimal; import java.sql.Time; import java.sql.Timestamp; -import java.util.Arrays; import java.util.Date; import java.util.UUID; import javax.cache.processor.EntryProcessor; @@ -81,37 +80,34 @@ public class VarintArraysSizeCompatibilityTest extends IgniteCompatibilityAbstra * @throws Exception If failed. */ public void testArraysStoringInCompatibilityMode() throws Exception { - startGrid(1, "2.1.0", new PostConfigurationClosure()); - - startGrid(2, "2.1.0", new PostConfigurationClosure()); + startGrid(1, "2.1.0", new CompatibilityConfigurationClosure()); + startGrid(2, "2.1.0", new CompatibilityConfigurationClosure()); Ignite ignite = startGrid(0); - CacheConfiguration cacheCfg = defaultCacheConfiguration(); + CacheConfiguration cacheCfg = defaultCacheConfiguration(); + cacheCfg.setCacheMode(CacheMode.REPLICATED); - cacheCfg.setName("varintTestCache"); cacheCfg.setRebalanceMode(CacheRebalanceMode.SYNC); - IgniteCache cache = ignite.createCache(cacheCfg); - - final TestObject obj = new TestObject(); - - final String key = "key"; + IgniteCache cache = ignite.createCache(cacheCfg); - cache.put(key, obj); + final TestObject val = new TestObject(); - cache.invoke( - key, - new EntryProcessor() { - @Override public TestObject process(MutableEntry entry, - Object... objects) throws EntryProcessorException { - TestObject val = entry.getValue(); + Integer key = backupKeys(cache, 1, 1000).get(0); - assertEquals(obj, val); + cache.put(key, val); - return val; - } - }); + assertTrue( + cache.invoke( + key, + new EntryProcessor() { + @Override public Boolean process(MutableEntry entry, + Object... objects) throws EntryProcessorException { + return val.equals(entry.getValue()); + } + }) + ); } /** */ @@ -135,23 +131,7 @@ private static class TestObject { /** {@inheritDoc} */ @Override public int hashCode() { - int result = Arrays.hashCode(bArr); - result = 31 * result + Arrays.hashCode(boolArr); - result = 31 * result + Arrays.hashCode(cArr); - result = 31 * result + Arrays.hashCode(sArr); - result = 31 * result + Arrays.hashCode(iArr); - result = 31 * result + Arrays.hashCode(lArr); - result = 31 * result + Arrays.hashCode(fArr); - result = 31 * result + Arrays.hashCode(dArr); - result = 31 * result + Arrays.hashCode(bdArr); - result = 31 * result + Arrays.hashCode(strArr); - result = 31 * result + Arrays.hashCode(uuidArr); - result = 31 * result + Arrays.hashCode(dateArr); - result = 31 * result + Arrays.hashCode(tsArr); - result = 31 * result + Arrays.hashCode(timeArr); - result = 31 * result + Arrays.hashCode(enumArr); - result = 31 * result + Arrays.hashCode(objArr); - return result; + return 0; } /** {@inheritDoc} */ @@ -166,7 +146,7 @@ private enum TestEnum { } /** */ - private static class PostConfigurationClosure implements IgniteInClosure { + private static class CompatibilityConfigurationClosure implements IgniteInClosure { /** {@inheritDoc} */ @Override public void apply(IgniteConfiguration cfg) { cfg.setLocalHost("127.0.0.1"); From 4af2a9259f446302df141fe05c016e0a15cc6d97 Mon Sep 17 00:00:00 2001 From: Vyacheslav Daradur Date: Thu, 10 Aug 2017 19:14:02 +0300 Subject: [PATCH 6/7] ignite-5736: added explanatory comment --- .../testframework/junits/CompatibilityTestIgniteNodeRunner.java | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/CompatibilityTestIgniteNodeRunner.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/CompatibilityTestIgniteNodeRunner.java index 8a6ff6d9deb3a..6a7e99d82135d 100644 --- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/CompatibilityTestIgniteNodeRunner.java +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/CompatibilityTestIgniteNodeRunner.java @@ -75,6 +75,7 @@ public static void main(String[] args) throws Exception { Ignite ignite = Ignition.start(cfg); + // it needs to set private static field 'ignite' of the IgniteNodeRunner class via reflection GridTestUtils.setFieldValue(new IgniteNodeRunner(), "ignite", ignite); } From 93530a8cd6af50326426abcee42f1490990c4fad Mon Sep 17 00:00:00 2001 From: Vyacheslav Daradur Date: Thu, 10 Aug 2017 20:03:30 +0300 Subject: [PATCH 7/7] ignite-5736: minor style code fixes --- .../junits/CompatibilityTestIgniteNodeRunner.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/CompatibilityTestIgniteNodeRunner.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/CompatibilityTestIgniteNodeRunner.java index 6a7e99d82135d..abd1afafc2a39 100644 --- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/CompatibilityTestIgniteNodeRunner.java +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/CompatibilityTestIgniteNodeRunner.java @@ -39,7 +39,7 @@ import org.apache.ignite.testframework.junits.multijvm.IgniteNodeRunner; /** - * Run Ignite node. + * Runs Ignite node. */ public class CompatibilityTestIgniteNodeRunner extends IgniteNodeRunner { /** */ @@ -112,19 +112,19 @@ public static void storeToFile(IgniteInClosure clos, String fileName) throws IOE /** * Reads closure from given file name and delete the file after. * - * @param closFileName Closure file name. + * @param fileName Closure file name. * @return IgniteInClosure for post-configuration. * @throws IOException In case of an error. * @see #storeToFile(IgniteInClosure, String) */ @SuppressWarnings("unchecked") public static IgniteInClosure readClosureFromFileAndDelete( - String closFileName) throws IOException { - try (BufferedReader closReader = Files.newBufferedReader(Paths.get(closFileName), StandardCharsets.UTF_8)) { - return (IgniteInClosure)new XStream().fromXML(closReader); + String fileName) throws IOException { + try (BufferedReader reader = Files.newBufferedReader(Paths.get(fileName), StandardCharsets.UTF_8)) { + return (IgniteInClosure)new XStream().fromXML(reader); } finally { - new File(closFileName).delete(); + new File(fileName).delete(); } } }