From 779b607a783bbbedca7b9b42a915e968aa4027eb Mon Sep 17 00:00:00 2001 From: Vyacheslav Daradur Date: Wed, 30 Aug 2017 14:40:59 +0300 Subject: [PATCH 1/4] IGNITE-5097 BinaryMarshaller should write ints in "varint" encoding where it makes sense --- .../apache/ignite/IgniteSystemProperties.java | 9 + .../internal/binary/BinaryEnumObjectImpl.java | 4 +- .../internal/binary/BinaryFieldImpl.java | 4 +- .../internal/binary/BinaryObjectImpl.java | 31 +- .../binary/BinaryObjectOffheapImpl.java | 18 +- .../BinarySerializedFieldComparator.java | 24 +- .../ignite/internal/binary/BinaryUtils.java | 294 ++++++++++++++++-- .../internal/binary/BinaryWriterExImpl.java | 136 ++++++-- .../binary/builder/BinaryBuilderReader.java | 78 +++-- .../builder/BinaryBuilderSerializer.java | 6 +- .../builder/BinaryEnumArrayLazyValue.java | 3 +- .../builder/BinaryObjectArrayLazyValue.java | 3 +- .../BinaryMarshallerIntArraysSizeTest.java | 41 +++ .../internal/binary/BinaryUtilsSelfTest.java | 88 ++++++ .../IgniteBinaryObjectsTestSuite.java | 4 + .../Binary/BinaryBuilderSelfTest.cs | 10 +- .../Binary/BinaryEqualityComparerTest.cs | 44 ++- .../Binary/BinarySelfTest.cs | 45 ++- .../Impl/Binary/BinaryObjectBuilder.cs | 22 +- .../Impl/Binary/BinaryUtils.cs | 166 +++++++--- 20 files changed, 857 insertions(+), 173 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerIntArraysSizeTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryUtilsSelfTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java index 39c19fbe7b1dd..21ced188c53db 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java @@ -712,6 +712,15 @@ public final class IgniteSystemProperties { */ public static final String IGNITE_DATA_STREAMING_EXECUTOR_SERVICE_TASKS_STEALING_THRESHOLD = "IGNITE_DATA_STREAMING_EXECUTOR_SERVICE_TASKS_STEALING_THRESHOLD"; + + /** + * When set to {@code true}, Ignite switches to compatibility mode with versions that writing + * length of arrays in default format. + * + * Default is {@code false}, which means that length of arrays will be written in varint encoding. + * Varint encoding description. + */ + public static final String IGNITE_NO_VARINT_ARRAY_LENGTH = "IGNITE_NO_VARINT_ARRAY_LENGTH"; /** * Enforces singleton. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java index 12a0fc352b99d..df8060f55c75b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java @@ -122,9 +122,9 @@ public BinaryEnumObjectImpl(BinaryContext ctx, byte[] arr) { if (this.typeId == GridBinaryMarshaller.UNREGISTERED_TYPE_ID) { assert arr[off] == GridBinaryMarshaller.STRING; - int len = BinaryPrimitives.readInt(arr, ++off); + int len = BinaryUtils.doReadArrayLength(arr, ++off); - off += 4; + off += BinaryUtils.sizeOfArrayLengthValue(len); byte[] bytes = BinaryPrimitives.readByteArray(arr, off, len); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldImpl.java index 59bd03dd39f0a..2cc0e96f21ede 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldImpl.java @@ -177,7 +177,7 @@ public int fieldId() { break; case GridBinaryMarshaller.STRING: { - int dataLen = buf.getInt(); + int dataLen = BinaryUtils.doReadArrayLength(buf); byte[] data = new byte[dataLen]; @@ -221,7 +221,7 @@ public int fieldId() { case GridBinaryMarshaller.DECIMAL: { int scale = buf.getInt(); - int dataLen = buf.getInt(); + int dataLen = BinaryUtils.doReadArrayLength(buf); byte[] data = new byte[dataLen]; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java index 1e706aedd1d86..bb1c4c1de6717 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java @@ -282,9 +282,11 @@ public void context(BinaryContext ctx) { assert arr[off] == GridBinaryMarshaller.STRING : arr[off]; - int len = BinaryPrimitives.readInt(arr, ++off); + int strLen = BinaryUtils.doReadArrayLength(arr, ++off); - String clsName = new String(arr, off + 4, len, UTF_8); + int len = BinaryUtils.sizeOfArrayLengthValue(strLen); + + String clsName = new String(arr, off + len, strLen, UTF_8); typeId = ctx.typeId(clsName); } @@ -334,9 +336,9 @@ public void context(BinaryContext ctx) { int typeId = BinaryPrimitives.readInt(arr, start + GridBinaryMarshaller.TYPE_ID_POS); if (typeId == GridBinaryMarshaller.UNREGISTERED_TYPE_ID) { - int len = BinaryPrimitives.readInt(arr, start + GridBinaryMarshaller.DFLT_HDR_LEN + 1); + int len = BinaryUtils.doReadArrayLength(arr, start + GridBinaryMarshaller.DFLT_HDR_LEN + 1); - return start + GridBinaryMarshaller.DFLT_HDR_LEN + len + 5; + return start + GridBinaryMarshaller.DFLT_HDR_LEN + 1 + BinaryUtils.sizeOfArrayLengthValue(len) + len; } else return start + GridBinaryMarshaller.DFLT_HDR_LEN; } @@ -423,9 +425,11 @@ else if (fieldOffLen == BinaryUtils.OFFSET_2) break; case GridBinaryMarshaller.STRING: { - int dataLen = BinaryPrimitives.readInt(arr, fieldPos + 1); + int dataLen = BinaryUtils.doReadArrayLength(arr, fieldPos + 1); + + int len = BinaryUtils.sizeOfArrayLengthValue(dataLen); - val = new String(arr, fieldPos + 5, dataLen, UTF_8); + val = new String(arr, fieldPos + 1 + len, dataLen, UTF_8); break; } @@ -470,9 +474,12 @@ else if (fieldOffLen == BinaryUtils.OFFSET_2) case GridBinaryMarshaller.DECIMAL: { int scale = BinaryPrimitives.readInt(arr, fieldPos + 1); + int len = 1 + 4; + + int dataLen = BinaryUtils.doReadArrayLength(arr, fieldPos + len); + len += BinaryUtils.sizeOfArrayLengthValue(dataLen); - int dataLen = BinaryPrimitives.readInt(arr, fieldPos + 5); - byte[] data = BinaryPrimitives.readByteArray(arr, fieldPos + 9, dataLen); + byte[] data = BinaryPrimitives.readByteArray(arr, fieldPos + len, dataLen); boolean negative = data[0] < 0; @@ -570,9 +577,9 @@ else if (fieldOffsetLen == BinaryUtils.OFFSET_2) break; case GridBinaryMarshaller.STRING: { - int dataLen = BinaryPrimitives.readInt(arr, fieldPos + 1); + int dataLen = BinaryUtils.doReadArrayLength(arr, fieldPos + 1); - totalLen = dataLen + 5; + totalLen = 1 + dataLen + BinaryUtils.sizeOfArrayLengthValue(dataLen); break; } @@ -588,9 +595,9 @@ else if (fieldOffsetLen == BinaryUtils.OFFSET_2) break; case GridBinaryMarshaller.DECIMAL: { - int dataLen = BinaryPrimitives.readInt(arr, fieldPos + 5); + int dataLen = BinaryUtils.doReadArrayLength(arr, fieldPos + 5); - totalLen = dataLen + 9; + totalLen = dataLen + 5 + BinaryUtils.sizeOfArrayLengthValue(dataLen); break; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java index 05f6963cabf71..daeca053421fd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java @@ -228,9 +228,9 @@ public BinaryObject heapCopy() { int typeId = BinaryPrimitives.readInt(ptr, start + GridBinaryMarshaller.TYPE_ID_POS); if (typeId == GridBinaryMarshaller.UNREGISTERED_TYPE_ID) { - int len = BinaryPrimitives.readInt(ptr, start + GridBinaryMarshaller.DFLT_HDR_LEN + 1); + int len = BinaryUtils.doReadArrayLength(ptr, start + GridBinaryMarshaller.DFLT_HDR_LEN + 1); - return start + GridBinaryMarshaller.DFLT_HDR_LEN + len + 5; + return start + GridBinaryMarshaller.DFLT_HDR_LEN + 1 + len + BinaryUtils.sizeOfArrayLengthValue(len); } else return start + GridBinaryMarshaller.DFLT_HDR_LEN; } @@ -317,8 +317,11 @@ else if (fieldOffLen == BinaryUtils.OFFSET_2) break; case GridBinaryMarshaller.STRING: { - int dataLen = BinaryPrimitives.readInt(ptr, fieldPos + 1); - byte[] data = BinaryPrimitives.readByteArray(ptr, fieldPos + 5, dataLen); + int dataLen = BinaryUtils.doReadArrayLength(ptr, fieldPos + 1); + + int len = BinaryUtils.sizeOfArrayLengthValue(dataLen); + + byte[] data = BinaryPrimitives.readByteArray(ptr, fieldPos + 1 + len, dataLen); val = new String(data, UTF_8); @@ -365,9 +368,12 @@ else if (fieldOffLen == BinaryUtils.OFFSET_2) case GridBinaryMarshaller.DECIMAL: { int scale = BinaryPrimitives.readInt(ptr, fieldPos + 1); + int len = 1 + 4; + + int dataLen = BinaryUtils.doReadArrayLength(ptr, fieldPos + len); + len += BinaryUtils.sizeOfArrayLengthValue(dataLen); - int dataLen = BinaryPrimitives.readInt(ptr, fieldPos + 5); - byte[] data = BinaryPrimitives.readByteArray(ptr, fieldPos + 9, dataLen); + byte[] data = BinaryPrimitives.readByteArray(ptr, fieldPos + len, dataLen); boolean negative = data[0] < 0; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinarySerializedFieldComparator.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinarySerializedFieldComparator.java index 43750732caca5..2db9bfdb05e90 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinarySerializedFieldComparator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinarySerializedFieldComparator.java @@ -180,6 +180,24 @@ private int readInt(int off) { return BinaryPrimitives.readInt(arr, curFieldPos + off); } + /** + * Reads value of length of an array, which can be presented in default format or varint encoding. + * Reading method depends whether this is offheap object and the constant {@link BinaryUtils#USE_VARINT_ARRAY_LENGTH}. + * Varint encoding description. + * + * If you need to know number of bytes which were used for storage of the read value, + * use the method {@link BinaryUtils#sizeOfArrayLengthValue(int)}. + * + * @param off Offset. + * @return Value of array's length. + */ + private int readArrayLength(int off) { + if (offheap()) + return BinaryUtils.doReadArrayLength(ptr, curFieldPos + off); + else + return BinaryUtils.doReadArrayLength(arr, curFieldPos + off); + } + /** * Read long value. * @@ -302,12 +320,12 @@ private static boolean isArray(@Nullable Object field) { */ private static boolean compareByteArrays(BinarySerializedFieldComparator c1, BinarySerializedFieldComparator c2, int off) { - int len = c1.readInt(off); + int len = c1.readArrayLength(off); - if (len != c2.readInt(off)) + if (len != c2.readArrayLength(off)) return false; else { - off += 4; + off += BinaryUtils.sizeOfArrayLengthValue(len); if (c1.offheap()) { if (c2.offheap()) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java index 8970a4eb15b84..3713818852517 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java @@ -28,6 +28,7 @@ import java.lang.reflect.Proxy; import java.math.BigDecimal; import java.math.BigInteger; +import java.nio.ByteBuffer; import java.sql.Time; import java.sql.Timestamp; import java.util.ArrayList; @@ -58,6 +59,7 @@ import org.apache.ignite.binary.BinaryRawWriter; import org.apache.ignite.binary.BinaryType; import org.apache.ignite.binary.Binarylizable; +import org.apache.ignite.internal.binary.builder.BinaryBuilderReader; import org.apache.ignite.internal.binary.builder.BinaryLazyValue; import org.apache.ignite.internal.binary.streams.BinaryInputStream; import org.apache.ignite.internal.processors.cache.CacheObjectByteArrayImpl; @@ -145,6 +147,10 @@ public class BinaryUtils { public static final boolean FIELDS_SORTED_ORDER = IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_BINARY_SORT_OBJECT_FIELDS); + /** Whether to write arrays lengths in varint encoding. */ + public static final boolean USE_VARINT_ARRAY_LENGTH = + !IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_NO_VARINT_ARRAY_LENGTH); + /** Field type names. */ private static final String[] FIELD_TYPE_NAMES; @@ -1199,7 +1205,7 @@ public static boolean isEnum(Class cls) { * @return Value. */ public static byte[] doReadByteArray(BinaryInputStream in) { - int len = in.readInt(); + int len = doReadArrayLength(in); return in.readByteArray(len); } @@ -1208,7 +1214,7 @@ public static byte[] doReadByteArray(BinaryInputStream in) { * @return Value. */ public static boolean[] doReadBooleanArray(BinaryInputStream in) { - int len = in.readInt(); + int len = doReadArrayLength(in); return in.readBooleanArray(len); } @@ -1217,7 +1223,7 @@ public static boolean[] doReadBooleanArray(BinaryInputStream in) { * @return Value. */ public static short[] doReadShortArray(BinaryInputStream in) { - int len = in.readInt(); + int len = doReadArrayLength(in); return in.readShortArray(len); } @@ -1226,7 +1232,7 @@ public static short[] doReadShortArray(BinaryInputStream in) { * @return Value. */ public static char[] doReadCharArray(BinaryInputStream in) { - int len = in.readInt(); + int len = doReadArrayLength(in); return in.readCharArray(len); } @@ -1235,7 +1241,7 @@ public static char[] doReadCharArray(BinaryInputStream in) { * @return Value. */ public static int[] doReadIntArray(BinaryInputStream in) { - int len = in.readInt(); + int len = doReadArrayLength(in); return in.readIntArray(len); } @@ -1244,7 +1250,7 @@ public static int[] doReadIntArray(BinaryInputStream in) { * @return Value. */ public static long[] doReadLongArray(BinaryInputStream in) { - int len = in.readInt(); + int len = doReadArrayLength(in); return in.readLongArray(len); } @@ -1253,7 +1259,7 @@ public static long[] doReadLongArray(BinaryInputStream in) { * @return Value. */ public static float[] doReadFloatArray(BinaryInputStream in) { - int len = in.readInt(); + int len = doReadArrayLength(in); return in.readFloatArray(len); } @@ -1262,7 +1268,7 @@ public static float[] doReadFloatArray(BinaryInputStream in) { * @return Value. */ public static double[] doReadDoubleArray(BinaryInputStream in) { - int len = in.readInt(); + int len = doReadArrayLength(in); return in.readDoubleArray(len); } @@ -1272,7 +1278,8 @@ public static double[] doReadDoubleArray(BinaryInputStream in) { */ public static BigDecimal doReadDecimal(BinaryInputStream in) { int scale = in.readInt(); - byte[] mag = doReadByteArray(in); + int magLen = doReadArrayLength(in); + byte[] mag = in.readByteArray(magLen); boolean negative = mag[0] < 0; @@ -1292,7 +1299,8 @@ public static BigDecimal doReadDecimal(BinaryInputStream in) { */ public static String doReadString(BinaryInputStream in) { if (!in.hasArray()) { - byte[] arr = doReadByteArray(in); + int len = doReadArrayLength(in); + byte[] arr = in.readByteArray(len); if (USE_STR_SERIALIZATION_VER_2) return utf8BytesToStr(arr, 0, arr.length); @@ -1300,7 +1308,7 @@ public static String doReadString(BinaryInputStream in) { return new String(arr, UTF_8); } - int strLen = in.readInt(); + int strLen = doReadArrayLength(in); int pos = in.position(); @@ -1363,7 +1371,7 @@ public static Time doReadTime(BinaryInputStream in) { * @throws BinaryObjectException In case of error. */ public static BigDecimal[] doReadDecimalArray(BinaryInputStream in) throws BinaryObjectException { - int len = in.readInt(); + int len = doReadArrayLength(in); BigDecimal[] arr = new BigDecimal[len]; @@ -1388,7 +1396,7 @@ public static BigDecimal[] doReadDecimalArray(BinaryInputStream in) throws Binar * @throws BinaryObjectException In case of error. */ public static String[] doReadStringArray(BinaryInputStream in) throws BinaryObjectException { - int len = in.readInt(); + int len = doReadArrayLength(in); String[] arr = new String[len]; @@ -1413,7 +1421,7 @@ public static String[] doReadStringArray(BinaryInputStream in) throws BinaryObje * @throws BinaryObjectException In case of error. */ public static UUID[] doReadUuidArray(BinaryInputStream in) throws BinaryObjectException { - int len = in.readInt(); + int len = doReadArrayLength(in); UUID[] arr = new UUID[len]; @@ -1438,7 +1446,7 @@ public static UUID[] doReadUuidArray(BinaryInputStream in) throws BinaryObjectEx * @throws BinaryObjectException In case of error. */ public static Date[] doReadDateArray(BinaryInputStream in) throws BinaryObjectException { - int len = in.readInt(); + int len = doReadArrayLength(in); Date[] arr = new Date[len]; @@ -1463,7 +1471,7 @@ public static Date[] doReadDateArray(BinaryInputStream in) throws BinaryObjectEx * @throws BinaryObjectException In case of error. */ public static Timestamp[] doReadTimestampArray(BinaryInputStream in) throws BinaryObjectException { - int len = in.readInt(); + int len = doReadArrayLength(in); Timestamp[] arr = new Timestamp[len]; @@ -1488,7 +1496,7 @@ public static Timestamp[] doReadTimestampArray(BinaryInputStream in) throws Bina * @throws BinaryObjectException In case of error. */ public static Time[] doReadTimeArray(BinaryInputStream in) throws BinaryObjectException { - int len = in.readInt(); + int len = doReadArrayLength(in); Time[] arr = new Time[len]; @@ -1524,7 +1532,10 @@ public static BinaryObject doReadBinaryObject(BinaryInputStream in, BinaryContex return new BinaryObjectOffheapImpl(ctx, in.offheapPointer() + pos, start, len); } else { - byte[] arr = doReadByteArray(in); + int len = in.readInt(); + + byte[] arr = in.readByteArray(len); + int start = in.readInt(); BinaryObjectImpl binO = new BinaryObjectImpl(ctx, arr, start); @@ -1714,7 +1725,7 @@ private static BinaryEnumObjectImpl doReadBinaryEnum(BinaryInputStream in, Binar * @return Enum array. */ private static Object[] doReadBinaryEnumArray(BinaryInputStream in, BinaryContext ctx) { - int len = in.readInt(); + int len = doReadArrayLength(in); Object[] arr = (Object[])Array.newInstance(BinaryObject.class, len); @@ -1753,7 +1764,7 @@ public static Enum doReadEnum(BinaryInputStream in, Class cls) throws Bina */ public static Object[] doReadEnumArray(BinaryInputStream in, BinaryContext ctx, ClassLoader ldr, Class cls) throws BinaryObjectException { - int len = in.readInt(); + int len = doReadArrayLength(in); Object[] arr = (Object[])Array.newInstance(cls, len); @@ -2011,7 +2022,7 @@ public static Object[] doReadObjectArray(BinaryInputStream in, BinaryContext ctx Class compType = doReadClass(in, ctx, ldr, deserialize); - int len = in.readInt(); + int len = doReadArrayLength(in); Object[] arr = deserialize ? (Object[])Array.newInstance(compType, len) : new Object[len]; @@ -2385,7 +2396,7 @@ else if (c > 0x07FF) { } else { arr[position++] = (byte)(0xC0 | ((c >> 6) & 0x1F)); - arr[position++] = (byte)(0x80 | (c & 0x3F)); + arr[position++] = (byte)(0x80 | (c & 0x3F)); } } @@ -2511,6 +2522,245 @@ public static Map mergeEnumValues(String typeName, return mergedMap; } + /** + * Reads from {@link BinaryInputStream} value of length of an array, + * which can be presented in default format or varint encoding. + * Reading method depends on the constant {@link #USE_VARINT_ARRAY_LENGTH}. + * Varint encoding description. + * + * If you need to know number of bytes which were used for storage of the read value, + * use the method {@link #sizeOfArrayLengthValue(int)}. + * + * @param in BinaryInputStream. + * @return Length of an array. + */ + public static int doReadArrayLength(BinaryInputStream in) { + if (!USE_VARINT_ARRAY_LENGTH) + return in.readInt(); + + return doReadUnsignedVarint(in); + } + + /** + * Reads from {@link BinaryBuilderReader} value of length of an array, + * which can be presented in default format or varint encoding. + * Reading method depends on the constant {@link #USE_VARINT_ARRAY_LENGTH}. + * Varint encoding description. + * + * If you need to know number of bytes which were used for storage of the read value, + * use the method {@link #sizeOfArrayLengthValue(int)}. + * + * @param in BinaryBuilderReader. + * @return Length of an array. + */ + public static int doReadArrayLength(BinaryBuilderReader in) { + if (!USE_VARINT_ARRAY_LENGTH) + return in.readInt(); + + return doReadUnsignedVarint(in); + } + + /** + * Reads from {@link ByteBuffer} value of length of an array, + * which can be presented in default format or varint encoding. + * Reading method depends on the constant {@link #USE_VARINT_ARRAY_LENGTH}. + * Varint encoding description. + * + * If you need to know number of bytes which were used for storage of the read value, + * use the method {@link #sizeOfArrayLengthValue(int)}. + * + * @param buf ByteBuffer. + * @return Length of an array. + */ + public static int doReadArrayLength(ByteBuffer buf) { + if (!USE_VARINT_ARRAY_LENGTH) + return buf.getInt(); + + return doReadUnsignedVarint(buf); + } + + /** + * Reads value of length of an array, which can be presented in default format or varint encoding. + * Starts reading from given offset. + * Reading method depends on the constant {@link #USE_VARINT_ARRAY_LENGTH}. + * Varint encoding description. + * + * If you need to know number of bytes which were used for storage of the read value, + * use the method {@link #sizeOfArrayLengthValue(int)}. + * + * @param arr Bytes array. + * @param off Offset. + * @return Length of an array. + */ + public static int doReadArrayLength(byte[] arr, int off) { + if (!USE_VARINT_ARRAY_LENGTH) + return BinaryPrimitives.readInt(arr, off); + + return doReadUnsignedVarint(arr, off); + } + + /** + * Reads value of length of an array, which can be presented in default format or varint encoding. + * Starts reading from given offset. + * Reading method depends on the constant {@link #USE_VARINT_ARRAY_LENGTH}. + * Varint encoding description. + * + * If you need to know number of bytes which were used for storage of the read value, + * use the method {@link #sizeOfArrayLengthValue(int)}. + * + * @param ptr Pointer. + * @param off Offset. + * @return Length of an array. + */ + public static int doReadArrayLength(long ptr, int off) { + if (!USE_VARINT_ARRAY_LENGTH) + return BinaryPrimitives.readInt(ptr, off); + + return doReadUnsignedVarint(ptr, off); + } + + /** + * Returns the amount of bytes required to write length of an array, + * which can be presented in default format or varint encoding. + * Varint encoding description. + * It depends on the constant {@link #USE_VARINT_ARRAY_LENGTH}. + * + * @param len Array length. + * @return Amount of bytes. + */ + public static int sizeOfArrayLengthValue(int len) { + if (!USE_VARINT_ARRAY_LENGTH) + return 4; + + return sizeInUnsignedVarint(len); + } + + /** + * Reads from {@link BinaryInputStream} integer value which is presented in varint encoding. + * Varint encoding description. + * + * @param in BinaryInputStream. + * @return Decoded integer value. + * @throws BinaryObjectException If have been read more than 5 bytes. + */ + public static int doReadUnsignedVarint(BinaryInputStream in) throws BinaryObjectException { + int val = doReadUnsignedVarint(in.array(), in.position()); + + in.position(in.position() + sizeInUnsignedVarint(val)); + + return val; + } + + /** + * Reads from {@link BinaryBuilderReader} integer value which is presented in varint encoding. + * Varint encoding description. + * + * @param in BinaryBuilderReader. + * @return Decoded integer value. + * @throws BinaryObjectException If have been read more than 5 bytes. + */ + public static int doReadUnsignedVarint(BinaryBuilderReader in) throws BinaryObjectException { + int val = doReadUnsignedVarint(in.array(), in.position()); + + in.position(in.position() + sizeInUnsignedVarint(val)); + + return val; + } + + /** + * Reads from given {@link ByteBuffer} integer value which is presented in varint encoding. + * Starts reading from given offset. + * Varint encoding description. + * + * @param buf ByteBuffer. + * @return Decoded integer value. + * @throws BinaryObjectException If have been read more than 5 bytes. + */ + public static int doReadUnsignedVarint(ByteBuffer buf) throws BinaryObjectException { + int val = doReadUnsignedVarint(buf.array(), buf.position()); + + buf.position(buf.position() + sizeInUnsignedVarint(val)); + + return val; + } + + /** + * Reads from given bytes array integer value which is presented in varint encoding. + * Starts reading from given offset. + * Varint encoding description. + * + * @param arr Bytes array. + * @param off Offset. + * @return Decoded integer value. + * @throws BinaryObjectException If have been read more than 5 bytes. + */ + public static int doReadUnsignedVarint(byte[] arr, int off) throws BinaryObjectException { + int val = 0; + int bits = 0; + int b; + + while (((b = arr[off++]) & 0x80) != 0) { + val |= (b & 0x7F) << bits; + bits += 7; + + if (bits > 35) + throw new BinaryObjectException("Varint reading failed, sequence length is too long"); + } + + return val | (b << bits); + } + + /** + * Reads via given pointer integer value which is presented in varint encoding. + * Starts reading from given offset. + * + * @param ptr Pointer. + * @param off Offset. + * @return Decoded integer value. + * @throws BinaryObjectException If have been read more than 5 bytes. + */ + public static int doReadUnsignedVarint(long ptr, int off) throws BinaryObjectException { + int val = 0; + int bits = 0; + int b; + + while (((b = BinaryPrimitives.readByte(ptr, off++)) & 0x80) != 0) { + val |= (b & 0x7F) << bits; + bits += 7; + + if (bits > 35) + throw new BinaryObjectException("Varint reading failed, sequence length is too long"); + } + + return val | (b << bits); + } + + /** + * Returns the encoded size of the given unsigned integer value. + * Varint encoding description. + * + * @param val Value to be encoded. + * @return Encoded size. + */ + public static int sizeInUnsignedVarint(int val) { + if (val < 0) + return 5; + + if (val <= Byte.MAX_VALUE) + return 1; + + if (val <= 16383) + return 2; + + if (val <= 2097151) + return 3; + + if (val <= 268435455) + return 4; + + return 5; + } + /** * Enum type. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java index a7f645c1f4763..0afeb41c8ffe8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java @@ -382,7 +382,7 @@ public void doWriteDecimal(@Nullable BigDecimal val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4 + 4); + out.unsafeEnsure(1 + 4 + 5); out.unsafeWriteByte(GridBinaryMarshaller.DECIMAL); @@ -400,7 +400,7 @@ public void doWriteDecimal(@Nullable BigDecimal val) { if (negative) vals[0] |= -0x80; - out.unsafeWriteInt(vals.length); + doUnsafeWriteArrayLength(vals.length); out.writeByteArray(vals); } } @@ -419,10 +419,10 @@ public void doWriteString(@Nullable String val) { else strArr = val.getBytes(UTF_8); - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.STRING); - out.unsafeWriteInt(strArr.length); + doUnsafeWriteArrayLength(strArr.length); out.writeByteArray(strArr); } } @@ -504,9 +504,9 @@ void doWriteByteArray(@Nullable byte[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.BYTE_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); out.writeByteArray(val); } @@ -519,9 +519,9 @@ void doWriteShortArray(@Nullable short[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.SHORT_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); out.writeShortArray(val); } @@ -534,14 +534,82 @@ void doWriteIntArray(@Nullable int[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.INT_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); out.writeIntArray(val); } } + /** + * Writes value of length of an array, which can be written in default format or varint encoding. + * Writing method depends on the constant {@link BinaryUtils#USE_VARINT_ARRAY_LENGTH}. + * Varint encoding description. + * + * If you need to know necessary number of bytes for writing, + * use the method {@link BinaryUtils#sizeOfArrayLengthValue(int)}. + * + * @param val Value to write. + */ + public void doUnsafeWriteArrayLength(int val) { + if (!BinaryUtils.USE_VARINT_ARRAY_LENGTH) + out.unsafeWriteInt(val); + else + doUnsafeWriteUnsignedVarint(val); + } + + /** + * Writes value of length of an array, which can be written in default format or varint encoding. + * Writing method depends on the constant {@link BinaryUtils#USE_VARINT_ARRAY_LENGTH}. + * Varint encoding description. + * + * If you need to know necessary number of bytes for writing, + * use the method {@link BinaryUtils#sizeOfArrayLengthValue(int)}. + * + * @param val Value to write. + */ + public void doWriteArrayLength(int val) { + if (!BinaryUtils.USE_VARINT_ARRAY_LENGTH) + out.writeInt(val); + else + doWriteUnsignedVarint(val); + } + + /** + * Writes integer value in varint encoding. + * Varint encoding description. + * Value must be positive. + * + * @param val Value to write. + */ + public void doWriteUnsignedVarint(int val) { + while ((val & 0xFFFFFF80) != 0) { + out.writeByte((byte)((val & 0x7F) | 0x80)); + val >>>= 7; + } + + out.writeByte((byte)(val & 0x7F)); + } + + /** + * Writes integer value in varint encoding. + * Uses unsafe writing methods. + * Before calling, make sure that {@link #out} has 5 bytes for writing. + * Varint encoding description. + * Value must be positive. + * + * @param val Value to write. + */ + public void doUnsafeWriteUnsignedVarint(int val) { + while ((val & 0xFFFFFF80) != 0) { + out.unsafeWriteByte((byte)((val & 0x7F) | 0x80)); + val >>>= 7; + } + + out.unsafeWriteByte((byte)(val & 0x7F)); + } + /** * @param val Long array. */ @@ -549,9 +617,9 @@ void doWriteLongArray(@Nullable long[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.LONG_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); out.writeLongArray(val); } @@ -564,9 +632,9 @@ void doWriteFloatArray(@Nullable float[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.FLOAT_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); out.writeFloatArray(val); } @@ -579,9 +647,9 @@ void doWriteDoubleArray(@Nullable double[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.DOUBLE_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); out.writeDoubleArray(val); } @@ -594,9 +662,9 @@ void doWriteCharArray(@Nullable char[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.CHAR_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); out.writeCharArray(val); } @@ -609,9 +677,9 @@ void doWriteBooleanArray(@Nullable boolean[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.BOOLEAN_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); out.writeBooleanArray(val); } @@ -624,9 +692,9 @@ void doWriteDecimalArray(@Nullable BigDecimal[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.DECIMAL_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); for (BigDecimal str : val) doWriteDecimal(str); @@ -640,9 +708,9 @@ void doWriteStringArray(@Nullable String[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.STRING_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); for (String str : val) doWriteString(str); @@ -656,9 +724,9 @@ void doWriteUuidArray(@Nullable UUID[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.UUID_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); for (UUID uuid : val) doWriteUuid(uuid); @@ -672,9 +740,9 @@ void doWriteDateArray(@Nullable Date[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.DATE_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); for (Date date : val) doWriteDate(date); @@ -688,9 +756,9 @@ void doWriteTimestampArray(@Nullable Timestamp[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.TIMESTAMP_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); for (Timestamp ts : val) doWriteTimestamp(ts); @@ -704,9 +772,9 @@ void doWriteTimeArray(@Nullable Time[] val) { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - out.unsafeEnsure(1 + 4); + out.unsafeEnsure(1 + 5); out.unsafeWriteByte(GridBinaryMarshaller.TIME_ARR); - out.unsafeWriteInt(val.length); + doUnsafeWriteArrayLength(val.length); for (Time time : val) doWriteTime(time); @@ -737,7 +805,7 @@ void doWriteObjectArray(@Nullable Object[] val) throws BinaryObjectException { doWriteString(val.getClass().getComponentType().getName()); } - out.writeInt(val.length); + doWriteArrayLength(val.length); for (Object obj : val) doWriteObject(obj); @@ -862,7 +930,7 @@ void doWriteEnumArray(@Nullable Object[] val) { doWriteString(val.getClass().getComponentType().getName()); } - out.writeInt(val.length); + doWriteArrayLength(val.length); // TODO: Denis: Redundant data for each element of the array. for (Object o : val) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java index 2d10cf4490e06..ca7f4f874635b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java @@ -180,7 +180,7 @@ public byte readBytePositioned(int pos) { * @return Read length of array. */ public int readLength() { - return BinaryPrimitives.readInt(arr, pos); + return BinaryUtils.doReadArrayLength(arr, pos); } /** @@ -189,7 +189,7 @@ public int readLength() { * @return String length. */ public int readStringLength() { - return BinaryPrimitives.readInt(arr, pos); + return BinaryUtils.doReadArrayLength(arr, pos); } /** @@ -206,7 +206,9 @@ public String readString() { if (flag != GridBinaryMarshaller.STRING) throw new BinaryObjectException("Failed to deserialize String."); - int len = readInt(); + int len = readStringLength(); + + pos += BinaryUtils.sizeOfArrayLengthValue(len); String str = new String(arr, pos, len, UTF_8); @@ -264,17 +266,21 @@ public void skipValue() { case GridBinaryMarshaller.BYTE_ARR: case GridBinaryMarshaller.BOOLEAN_ARR: - len = 4 + readLength(); + len = readLength(); + len += BinaryUtils.sizeOfArrayLengthValue(len); break; case GridBinaryMarshaller.STRING: - len = 4 + readStringLength(); + len = readStringLength(); + len += BinaryUtils.sizeOfArrayLengthValue(len); break; case GridBinaryMarshaller.DECIMAL: - len = /** scale */ 4 + /** mag len */ 4 + /** mag bytes count */ readInt(4); + int magLen = BinaryUtils.doReadArrayLength(arr, pos); + + len = /* scale */ 4 + BinaryUtils.sizeOfArrayLengthValue(magLen) + magLen; break; @@ -300,19 +306,22 @@ public void skipValue() { case GridBinaryMarshaller.CHAR_ARR: case GridBinaryMarshaller.SHORT_ARR: - len = 4 + readLength() * 2; + len = readLength(); + len = len * 2 + BinaryUtils.sizeOfArrayLengthValue(len); break; case GridBinaryMarshaller.INT_ARR: case GridBinaryMarshaller.FLOAT_ARR: - len = 4 + readLength() * 4; + len = readLength(); + len = len * 4 + BinaryUtils.sizeOfArrayLengthValue(len); break; case GridBinaryMarshaller.LONG_ARR: case GridBinaryMarshaller.DOUBLE_ARR: - len = 4 + readLength() * 8; + len = readLength(); + len = len * 8 + BinaryUtils.sizeOfArrayLengthValue(len); break; @@ -324,7 +333,7 @@ public void skipValue() { case GridBinaryMarshaller.ENUM_ARR: case GridBinaryMarshaller.UUID_ARR: case GridBinaryMarshaller.STRING_ARR: { - int size = readInt(); + int size = BinaryUtils.doReadArrayLength(this); for (int i = 0; i < size; i++) skipValue(); @@ -580,12 +589,15 @@ public Object parseValue() { return arr[pos++] != 0; case GridBinaryMarshaller.DECIMAL: - plainLazyValLen = /** scale */ 4 + /** mag len */ 4 + /** mag bytes count */ readInt(4); + int magLen = BinaryUtils.doReadArrayLength(arr, pos + 4); + + plainLazyValLen = /* scale */ 4 + BinaryUtils.sizeOfArrayLengthValue(magLen) + magLen; break; case GridBinaryMarshaller.STRING: - plainLazyValLen = 4 + readStringLength(); + plainLazyValLen = readStringLength(); + plainLazyValLen += BinaryUtils.sizeOfArrayLengthValue(plainLazyValLen); break; @@ -610,49 +622,57 @@ public Object parseValue() { break; case GridBinaryMarshaller.BYTE_ARR: - plainLazyValLen = 4 + readLength(); + plainLazyValLen = readLength(); + plainLazyValLen += BinaryUtils.sizeOfArrayLengthValue(plainLazyValLen); modifiableLazyVal = true; break; case GridBinaryMarshaller.SHORT_ARR: - plainLazyValLen = 4 + readLength() * 2; + plainLazyValLen = readLength(); + plainLazyValLen = plainLazyValLen * 2 + BinaryUtils.sizeOfArrayLengthValue(plainLazyValLen); modifiableLazyVal = true; break; case GridBinaryMarshaller.INT_ARR: - plainLazyValLen = 4 + readLength() * 4; + plainLazyValLen = readLength(); + plainLazyValLen = plainLazyValLen * 4 + BinaryUtils.sizeOfArrayLengthValue(plainLazyValLen); modifiableLazyVal = true; break; case GridBinaryMarshaller.LONG_ARR: - plainLazyValLen = 4 + readLength() * 8; + plainLazyValLen = readLength(); + plainLazyValLen = plainLazyValLen * 8 + BinaryUtils.sizeOfArrayLengthValue(plainLazyValLen); modifiableLazyVal = true; break; case GridBinaryMarshaller.FLOAT_ARR: - plainLazyValLen = 4 + readLength() * 4; + plainLazyValLen = readLength(); + plainLazyValLen = plainLazyValLen * 4 + BinaryUtils.sizeOfArrayLengthValue(plainLazyValLen); modifiableLazyVal = true; break; case GridBinaryMarshaller.DOUBLE_ARR: - plainLazyValLen = 4 + readLength() * 8; + plainLazyValLen = readLength(); + plainLazyValLen = plainLazyValLen * 8 + BinaryUtils.sizeOfArrayLengthValue(plainLazyValLen); modifiableLazyVal = true; break; case GridBinaryMarshaller.CHAR_ARR: - plainLazyValLen = 4 + readLength() * 2; + plainLazyValLen = readLength(); + plainLazyValLen = plainLazyValLen * 2 + BinaryUtils.sizeOfArrayLengthValue(plainLazyValLen); modifiableLazyVal = true; break; case GridBinaryMarshaller.BOOLEAN_ARR: - plainLazyValLen = 4 + readLength(); + plainLazyValLen = readLength(); + plainLazyValLen += BinaryUtils.sizeOfArrayLengthValue(plainLazyValLen); modifiableLazyVal = true; break; @@ -661,7 +681,7 @@ public Object parseValue() { return new BinaryObjectArrayLazyValue(this); case GridBinaryMarshaller.DATE_ARR: { - int size = readInt(); + int size = BinaryUtils.doReadArrayLength(this); Date[] res = new Date[size]; @@ -684,7 +704,7 @@ public Object parseValue() { } case GridBinaryMarshaller.TIMESTAMP_ARR: { - int size = readInt(); + int size = BinaryUtils.doReadArrayLength(this); Timestamp[] res = new Timestamp[size]; @@ -716,7 +736,7 @@ public Object parseValue() { } case GridBinaryMarshaller.TIME_ARR: { - int size = readInt(); + int size = BinaryUtils.doReadArrayLength(this); Time[] res = new Time[size]; @@ -741,18 +761,22 @@ public Object parseValue() { case GridBinaryMarshaller.UUID_ARR: case GridBinaryMarshaller.STRING_ARR: case GridBinaryMarshaller.DECIMAL_ARR: { - int size = readInt(); + int size = BinaryUtils.doReadArrayLength(this); for (int i = 0; i < size; i++) { byte flag = arr[pos++]; if (flag == GridBinaryMarshaller.UUID) pos += 8 + 8; - else if (flag == GridBinaryMarshaller.STRING) - pos += 4 + readStringLength(); + else if (flag == GridBinaryMarshaller.STRING) { + int strLen = readStringLength(); + pos += BinaryUtils.sizeOfArrayLengthValue(strLen); + pos += strLen; + } else if (flag == GridBinaryMarshaller.DECIMAL) { pos += 4; // scale value - pos += 4 + readLength(); + int len = BinaryUtils.doReadArrayLength(arr, pos); + pos += BinaryUtils.sizeOfArrayLengthValue(len) + len; } else assert flag == GridBinaryMarshaller.NULL; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java index 018444c65123a..6d12ececb4740 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java @@ -191,7 +191,7 @@ public void writeValue(BinaryWriterExImpl writer, Object val, boolean forceCol, writer.writeByte(GridBinaryMarshaller.ENUM_ARR); writer.writeInt(compTypeId); - writer.writeInt(enumArr.length); + writer.doUnsafeWriteArrayLength(enumArr.length); for (Enum anEnum : enumArr) writeValue(writer, anEnum); @@ -216,7 +216,7 @@ public void writeValue(BinaryWriterExImpl writer, Object val, boolean forceCol, public void writeArray(BinaryWriterExImpl writer, byte elementType, Object[] arr, int compTypeId) { writer.writeByte(elementType); writer.writeInt(compTypeId); - writer.writeInt(arr.length); + writer.doUnsafeWriteArrayLength(arr.length); for (Object obj : arr) writeValue(writer, obj); @@ -232,7 +232,7 @@ public void writeArray(BinaryWriterExImpl writer, byte elementType, Object[] arr writer.writeByte(elementType); writer.writeInt(GridBinaryMarshaller.UNREGISTERED_TYPE_ID); writer.writeString(clsName); - writer.writeInt(arr.length); + writer.doUnsafeWriteArrayLength(arr.length); for (Object obj : arr) writeValue(writer, obj); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryEnumArrayLazyValue.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryEnumArrayLazyValue.java index 787ff638b995e..fe1bb88e6e617 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryEnumArrayLazyValue.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryEnumArrayLazyValue.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.binary.builder; +import org.apache.ignite.internal.binary.BinaryUtils; import org.apache.ignite.internal.binary.BinaryWriterExImpl; import org.apache.ignite.internal.binary.GridBinaryMarshaller; import org.apache.ignite.internal.util.typedef.internal.U; @@ -78,7 +79,7 @@ protected BinaryEnumArrayLazyValue(BinaryBuilderReader reader) { //skipping component type id reader.readInt(); - int size = reader.readInt(); + int size = BinaryUtils.doReadArrayLength(reader); BinaryBuilderEnum[] res = new BinaryBuilderEnum[size]; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectArrayLazyValue.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectArrayLazyValue.java index 8962107c77ac8..7ff2786195053 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectArrayLazyValue.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectArrayLazyValue.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.binary.builder; +import org.apache.ignite.internal.binary.BinaryUtils; import org.apache.ignite.internal.binary.GridBinaryMarshaller; import org.apache.ignite.internal.binary.BinaryWriterExImpl; import org.apache.ignite.internal.util.typedef.internal.U; @@ -62,7 +63,7 @@ protected BinaryObjectArrayLazyValue(BinaryBuilderReader reader) { clsName = null; } - int size = reader.readInt(); + int size = BinaryUtils.doReadArrayLength(reader); lazyValsArr = new Object[size]; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerIntArraysSizeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerIntArraysSizeTest.java new file mode 100644 index 0000000000000..fa6426db3aad5 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerIntArraysSizeTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.binary; + +import org.apache.ignite.IgniteSystemProperties; + +/** + * {@inheritDoc} + * + * Tests of Ignite compatibility mode with versions that writing length of arrays in default format. + */ +public class BinaryMarshallerIntArraysSizeTest extends BinaryMarshallerSelfTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + System.setProperty(IgniteSystemProperties.IGNITE_NO_VARINT_ARRAY_LENGTH, "true"); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + super.afterTestsStopped(); + + System.clearProperty(IgniteSystemProperties.IGNITE_NO_VARINT_ARRAY_LENGTH); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryUtilsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryUtilsSelfTest.java new file mode 100644 index 0000000000000..9bcacfbc667c9 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryUtilsSelfTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.binary; + +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.binary.streams.BinaryHeapInputStream; +import org.apache.ignite.logger.NullLogger; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * Tests for {@link BinaryUtils}. + */ +public class BinaryUtilsSelfTest extends GridCommonAbstractTest { + /** */ + private int[] ints; + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + ints = new int[] {Integer.MIN_VALUE, Short.MIN_VALUE, -90_000, Byte.MIN_VALUE, -5678, -234, -11, 0, 10, 200, 3000, Byte.MAX_VALUE, Short.MAX_VALUE, 40_000, 123_545, Integer.MAX_VALUE}; + } + + /** + * @throws Exception If failed. + */ + public void testUnsignedVarint() throws Exception { + BinaryWriterExImpl writer = createWriter(); + + int len = 0; + + for (int i = 0; i < ints.length; i++) { + writer.doWriteUnsignedVarint(ints[i]); + + assertEquals((len += BinaryUtils.sizeInUnsignedVarint(ints[i])), writer.array().length); + } + + BinaryHeapInputStream in = new BinaryHeapInputStream(writer.array()); + + for (int i = 0; i < ints.length; i++) + assertEquals(ints[i], BinaryUtils.doReadUnsignedVarint(in)); + } + + /** + * @throws Exception If failed. + */ + public void testSizeInUnsignedVarint() throws Exception { + assertEquals(1, BinaryUtils.sizeInUnsignedVarint(0)); + assertEquals(1, BinaryUtils.sizeInUnsignedVarint(Byte.MAX_VALUE)); + assertEquals(2, BinaryUtils.sizeInUnsignedVarint(Byte.MAX_VALUE + 1)); + + assertEquals(2, BinaryUtils.sizeInUnsignedVarint(128 * 128 - 1)); + assertEquals(3, BinaryUtils.sizeInUnsignedVarint(128 * 128)); + + assertEquals(3, BinaryUtils.sizeInUnsignedVarint(128 * 128 * 128 - 1)); + assertEquals(4, BinaryUtils.sizeInUnsignedVarint(128 * 128 * 128)); + + assertEquals(4, BinaryUtils.sizeInUnsignedVarint(128 * 128 * 128 * 128 - 1)); + assertEquals(5, BinaryUtils.sizeInUnsignedVarint(128 * 128 * 128 * 128)); + assertEquals(5, BinaryUtils.sizeInUnsignedVarint(Integer.MAX_VALUE)); + + // negative values + assertEquals(5, BinaryUtils.sizeInUnsignedVarint(-1)); + assertEquals(5, BinaryUtils.sizeInUnsignedVarint(Byte.MIN_VALUE)); + assertEquals(5, BinaryUtils.sizeInUnsignedVarint(Short.MIN_VALUE)); + assertEquals(5, BinaryUtils.sizeInUnsignedVarint(Integer.MIN_VALUE)); + } + + /** */ + private BinaryWriterExImpl createWriter() { + return new BinaryWriterExImpl(new BinaryContext(BinaryCachingMetadataHandler.create(), new IgniteConfiguration(), new NullLogger())); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java index c0211be4adbd7..92d5baeb8c5ea 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java @@ -28,6 +28,7 @@ import org.apache.ignite.internal.binary.BinaryFieldsOffheapSelfTest; import org.apache.ignite.internal.binary.BinaryFooterOffsetsHeapSelfTest; import org.apache.ignite.internal.binary.BinaryFooterOffsetsOffheapSelfTest; +import org.apache.ignite.internal.binary.BinaryMarshallerIntArraysSizeTest; import org.apache.ignite.internal.binary.BinaryMarshallerSelfTest; import org.apache.ignite.internal.binary.BinaryObjectBuilderAdditionalSelfTest; import org.apache.ignite.internal.binary.BinaryObjectBuilderDefaultMappersSelfTest; @@ -37,6 +38,7 @@ import org.apache.ignite.internal.binary.BinarySerialiedFieldComparatorSelfTest; import org.apache.ignite.internal.binary.BinarySimpleNameTestPropertySelfTest; import org.apache.ignite.internal.binary.BinaryTreeSelfTest; +import org.apache.ignite.internal.binary.BinaryUtilsSelfTest; import org.apache.ignite.internal.binary.GridBinaryAffinityKeySelfTest; import org.apache.ignite.internal.binary.GridBinaryMarshallerCtxDisabledSelfTest; import org.apache.ignite.internal.binary.GridBinaryWildcardsSelfTest; @@ -89,6 +91,8 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(BinaryTreeSelfTest.class); suite.addTestSuite(BinaryMarshallerSelfTest.class); + suite.addTestSuite(BinaryMarshallerIntArraysSizeTest.class); + suite.addTestSuite(BinaryUtilsSelfTest.class); suite.addTestSuite(BinaryObjectExceptionSelfTest.class); suite.addTestSuite(BinarySerialiedFieldComparatorSelfTest.class); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryBuilderSelfTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryBuilderSelfTest.cs index 61f90a3aa0535..cc70585275a6a 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryBuilderSelfTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryBuilderSelfTest.cs @@ -616,7 +616,15 @@ public void TestEquality() Assert.AreEqual(obj1, obj2); - Assert.AreEqual(-88648479, obj1.GetHashCode()); + if (BinaryUtils.UseVarintArrayLenght) + { + Assert.AreEqual(109523595, obj1.GetHashCode()); + } + else + { + Assert.AreEqual(-88648479, obj1.GetHashCode()); + } + Assert.AreEqual(obj1.GetHashCode(), obj2.GetHashCode()); Assert.AreEqual("myType [, int=1, str=foo]", Regex.Replace(obj1.ToString(), "idHash=\\d+", "")); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryEqualityComparerTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryEqualityComparerTest.cs index 4dc4d93923fd7..461a37fdfaa6d 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryEqualityComparerTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryEqualityComparerTest.cs @@ -102,22 +102,46 @@ public void TestBinaryObjects() Assert.IsFalse(BinaryArrayEqualityComparer.Equals(obj5, obj6)); // BinaryObject.GetHashCode. - Assert.AreEqual(1934949494, obj1.GetHashCode()); - Assert.AreEqual(-2013102781, obj2.GetHashCode()); - Assert.AreEqual(1424415317, obj3.GetHashCode()); - Assert.AreEqual(1771330338, obj4.GetHashCode()); + if (BinaryUtils.UseVarintArrayLenght) + { + Assert.AreEqual(860484896, obj1.GetHashCode()); + Assert.AreEqual(1207399917, obj2.GetHashCode()); + Assert.AreEqual(567081889, obj3.GetHashCode()); + Assert.AreEqual(913996910, obj4.GetHashCode()); + Assert.AreEqual(860514687, BinaryArrayEqualityComparer.GetHashCode(obj6)); + } + else + { + Assert.AreEqual(1934949494, obj1.GetHashCode()); + Assert.AreEqual(-2013102781, obj2.GetHashCode()); + Assert.AreEqual(1424415317, obj3.GetHashCode()); + Assert.AreEqual(1771330338, obj4.GetHashCode()); + Assert.AreEqual(1934979285, BinaryArrayEqualityComparer.GetHashCode(obj6)); + } + Assert.AreEqual(obj1.GetHashCode(), obj5.GetHashCode()); - Assert.AreEqual(1934979285, BinaryArrayEqualityComparer.GetHashCode(obj6)); // Comparer.GetHashCode. + if (BinaryUtils.UseVarintArrayLenght) + { + Assert.AreEqual(860484896, BinaryArrayEqualityComparer.GetHashCode(obj1)); + Assert.AreEqual(1207399917, BinaryArrayEqualityComparer.GetHashCode(obj2)); + Assert.AreEqual(567081889, BinaryArrayEqualityComparer.GetHashCode(obj3)); + Assert.AreEqual(913996910, BinaryArrayEqualityComparer.GetHashCode(obj4)); + Assert.AreEqual(860514687, BinaryArrayEqualityComparer.GetHashCode(obj6)); + } + else + { + Assert.AreEqual(1934949494, BinaryArrayEqualityComparer.GetHashCode(obj1)); + Assert.AreEqual(-2013102781, BinaryArrayEqualityComparer.GetHashCode(obj2)); + Assert.AreEqual(1424415317, BinaryArrayEqualityComparer.GetHashCode(obj3)); + Assert.AreEqual(1771330338, BinaryArrayEqualityComparer.GetHashCode(obj4)); + Assert.AreEqual(1934979285, BinaryArrayEqualityComparer.GetHashCode(obj6)); + } + Assert.AreEqual(2001751043, BinaryArrayEqualityComparer.GetHashCode(GetBinaryObject(0, null, 0))); Assert.AreEqual(194296580, BinaryArrayEqualityComparer.GetHashCode(GetBinaryObject(1, null, 0))); - Assert.AreEqual(1934949494, BinaryArrayEqualityComparer.GetHashCode(obj1)); - Assert.AreEqual(-2013102781, BinaryArrayEqualityComparer.GetHashCode(obj2)); - Assert.AreEqual(1424415317, BinaryArrayEqualityComparer.GetHashCode(obj3)); - Assert.AreEqual(1771330338, BinaryArrayEqualityComparer.GetHashCode(obj4)); Assert.AreEqual(BinaryArrayEqualityComparer.GetHashCode(obj1), BinaryArrayEqualityComparer.GetHashCode(obj5)); - Assert.AreEqual(1934979285, BinaryArrayEqualityComparer.GetHashCode(obj6)); // GetHashCode consistency. foreach (var obj in new[] {obj1, obj2, obj3, obj4, obj5, obj6}) diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinarySelfTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinarySelfTest.cs index e24dca0ddb5cc..9083707432678 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinarySelfTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinarySelfTest.cs @@ -627,7 +627,26 @@ public void TestWriteEnumArrayRegistered() Assert.AreEqual(vals, newVals); } - + + /** + * Checks the writing an integer value in varint encoding. + */ + [Test] + public void TestWriteUvarint() + { + int[] vals = {int.MinValue, short.MinValue, -16384, -128, 0, 127, 16383, short.MaxValue, int.MaxValue}; + + var stream = new BinaryHeapStream(64); + + foreach (int val in vals) + BinaryUtils.WriteUvarint(val, stream); + + stream.Seek(0, SeekOrigin.Begin); + + foreach (int val in vals) + Assert.AreEqual(val, BinaryUtils.ReadUvarint(stream)); + } + /// /// Test object with dates. /// @@ -2619,5 +2638,29 @@ public override int GetHashCode() return !left.Equals(right); } } + + /** + * Tests Ignite compatibility mode with versions that writing length of arrays in default format. + */ + [Test] + public void TestNoVarintArrayLengthMode() + { + if (Environment.GetEnvironmentVariable(BinaryUtils.IgniteNoVarintArrayLength) != "true") + { + // Run "TestNoVarintArrayLengthMode" in a separate process with changed setting. + Environment.SetEnvironmentVariable(BinaryUtils.IgniteNoVarintArrayLength, "true"); + + TestUtils.RunTestInNewProcess(GetType().FullName, "TestNoVarintArrayLengthMode"); + } + } + + /** + * Test tear down. + */ + [TearDown] + public void TearDown() + { + Environment.SetEnvironmentVariable(BinaryUtils.IgniteNoVarintArrayLength, null); + } } } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectBuilder.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectBuilder.cs index c310b3afa4064..3d8de5dac2cea 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectBuilder.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectBuilder.cs @@ -869,9 +869,9 @@ internal void ProcessBuilder(IBinaryStream outStream, BinaryObjectBuilder builde case BinaryUtils.TypeDecimal: TransferBytes(inStream, outStream, 4); // Transfer scale - int magLen = inStream.ReadInt(); // Transfer magnitude length. + int magLen = BinaryUtils.ReadArrayLength(inStream); // Transfer magnitude length. - outStream.WriteInt(magLen); + BinaryUtils.WriteArrayLength(magLen, outStream); TransferBytes(inStream, outStream, magLen); // Transfer magnitude. @@ -936,9 +936,9 @@ internal void ProcessBuilder(IBinaryStream outStream, BinaryObjectBuilder builde case BinaryUtils.TypeArrayString: case BinaryUtils.TypeArrayGuid: case BinaryUtils.TypeArrayTimestamp: - int arrLen = inStream.ReadInt(); + int arrLen = BinaryUtils.ReadArrayLength(inStream); - outStream.WriteInt(arrLen); + BinaryUtils.WriteArrayLength(arrLen, outStream); for (int i = 0; i < arrLen; i++) Mutate0(ctx, inStream, outStream, false, null); @@ -958,9 +958,9 @@ internal void ProcessBuilder(IBinaryStream outStream, BinaryObjectBuilder builde BinaryUtils.WriteString(BinaryUtils.ReadString(inStream), outStream); // String data. } - arrLen = inStream.ReadInt(); + arrLen = BinaryUtils.ReadArrayLength(inStream); - outStream.WriteInt(arrLen); + BinaryUtils.WriteArrayLength(arrLen, outStream); for (int i = 0; i < arrLen; i++) Mutate0(ctx, inStream, outStream, false, EmptyVals); @@ -995,7 +995,11 @@ internal void ProcessBuilder(IBinaryStream outStream, BinaryObjectBuilder builde break; case BinaryUtils.TypeBinary: - TransferArray(inStream, outStream, 1); // Data array. + int len = inStream.ReadInt(); + + outStream.WriteInt(len); + + TransferBytes(inStream, outStream, len); // Data array. TransferBytes(inStream, outStream, 4); // Offset in array. break; @@ -1034,9 +1038,9 @@ private static void TransferBytes(BinaryHeapStream inStream, IBinaryStream outSt private static void TransferArray(BinaryHeapStream inStream, IBinaryStream outStream, int elemSize) { - int len = inStream.ReadInt(); + int len = BinaryUtils.ReadArrayLength(inStream); - outStream.WriteInt(len); + BinaryUtils.WriteArrayLength(len, outStream); TransferBytes(inStream, outStream, elemSize * len); } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs index 91a536edc7438..98a2b8383dde6 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryUtils.cs @@ -252,6 +252,13 @@ internal static class BinaryUtils public static readonly bool UseStringSerializationVer2 = (Environment.GetEnvironmentVariable(IgniteBinaryMarshallerUseStringSerializationVer2) ?? "false") == "true"; + /** Length of arrays format environment variable. */ + public const string IgniteNoVarintArrayLength = "IGNITE_NO_VARINT_ARRAY_LENGTH"; + + /** Length of arrays writing mode. */ + public static readonly bool UseVarintArrayLenght = + (Environment.GetEnvironmentVariable(IgniteNoVarintArrayLength) ?? "false") != "true"; + /// /// Default marshaller. /// @@ -267,7 +274,7 @@ public static Marshaller Marshaller */ public static void WriteBooleanArray(bool[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); stream.WriteBoolArray(vals); } @@ -279,7 +286,7 @@ public static void WriteBooleanArray(bool[] vals, IBinaryStream stream) */ public static bool[] ReadBooleanArray(IBinaryStream stream) { - int len = stream.ReadInt(); + int len = ReadArrayLength(stream); return stream.ReadBoolArray(len); } @@ -292,7 +299,7 @@ public static bool[] ReadBooleanArray(IBinaryStream stream) */ public static void WriteByteArray(byte[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); stream.WriteByteArray(vals); } @@ -304,7 +311,9 @@ public static void WriteByteArray(byte[] vals, IBinaryStream stream) */ public static byte[] ReadByteArray(IBinaryStream stream) { - return stream.ReadByteArray(stream.ReadInt()); + int len = ReadArrayLength(stream); + + return stream.ReadByteArray(len); } /** @@ -314,7 +323,7 @@ public static byte[] ReadByteArray(IBinaryStream stream) */ public static unsafe sbyte[] ReadSbyteArray(IBinaryStream stream) { - int len = stream.ReadInt(); + int len = ReadArrayLength(stream); sbyte[] res = new sbyte[len]; @@ -333,7 +342,7 @@ public static unsafe sbyte[] ReadSbyteArray(IBinaryStream stream) */ public static void WriteShortArray(short[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); stream.WriteShortArray(vals); } @@ -345,7 +354,7 @@ public static void WriteShortArray(short[] vals, IBinaryStream stream) */ public static unsafe ushort[] ReadUshortArray(IBinaryStream stream) { - int len = stream.ReadInt(); + int len = ReadArrayLength(stream); ushort[] res = new ushort[len]; @@ -364,7 +373,9 @@ public static unsafe ushort[] ReadUshortArray(IBinaryStream stream) */ public static short[] ReadShortArray(IBinaryStream stream) { - return stream.ReadShortArray(stream.ReadInt()); + int len = ReadArrayLength(stream); + + return stream.ReadShortArray(len); } /** @@ -374,7 +385,7 @@ public static short[] ReadShortArray(IBinaryStream stream) */ public static void WriteIntArray(int[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); stream.WriteIntArray(vals); } @@ -386,7 +397,9 @@ public static void WriteIntArray(int[] vals, IBinaryStream stream) */ public static int[] ReadIntArray(IBinaryStream stream) { - return stream.ReadIntArray(stream.ReadInt()); + int len = ReadArrayLength(stream); + + return stream.ReadIntArray(len); } /** @@ -396,7 +409,7 @@ public static int[] ReadIntArray(IBinaryStream stream) */ public static unsafe uint[] ReadUintArray(IBinaryStream stream) { - int len = stream.ReadInt(); + int len = ReadArrayLength(stream); uint[] res = new uint[len]; @@ -415,7 +428,7 @@ public static unsafe uint[] ReadUintArray(IBinaryStream stream) */ public static void WriteLongArray(long[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); stream.WriteLongArray(vals); } @@ -427,7 +440,9 @@ public static void WriteLongArray(long[] vals, IBinaryStream stream) */ public static long[] ReadLongArray(IBinaryStream stream) { - return stream.ReadLongArray(stream.ReadInt()); + int len = ReadArrayLength(stream); + + return stream.ReadLongArray(len); } /** @@ -437,7 +452,7 @@ public static long[] ReadLongArray(IBinaryStream stream) */ public static unsafe ulong[] ReadUlongArray(IBinaryStream stream) { - int len = stream.ReadInt(); + int len = ReadArrayLength(stream); ulong[] res = new ulong[len]; @@ -456,7 +471,7 @@ public static unsafe ulong[] ReadUlongArray(IBinaryStream stream) */ public static void WriteCharArray(char[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); stream.WriteCharArray(vals); } @@ -468,7 +483,7 @@ public static void WriteCharArray(char[] vals, IBinaryStream stream) */ public static char[] ReadCharArray(IBinaryStream stream) { - int len = stream.ReadInt(); + int len = ReadArrayLength(stream); return stream.ReadCharArray(len); } @@ -480,7 +495,7 @@ public static char[] ReadCharArray(IBinaryStream stream) */ public static void WriteFloatArray(float[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); stream.WriteFloatArray(vals); } @@ -492,7 +507,7 @@ public static void WriteFloatArray(float[] vals, IBinaryStream stream) */ public static float[] ReadFloatArray(IBinaryStream stream) { - int len = stream.ReadInt(); + int len = ReadArrayLength(stream); return stream.ReadFloatArray(len); } @@ -504,7 +519,7 @@ public static float[] ReadFloatArray(IBinaryStream stream) */ public static void WriteDoubleArray(double[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); stream.WriteDoubleArray(vals); } @@ -516,7 +531,7 @@ public static void WriteDoubleArray(double[] vals, IBinaryStream stream) */ public static double[] ReadDoubleArray(IBinaryStream stream) { - int len = stream.ReadInt(); + int len = ReadArrayLength(stream); return stream.ReadDoubleArray(len); } @@ -557,7 +572,7 @@ public static void WriteTimestamp(DateTime val, IBinaryStream stream) /// Stream. public static void WriteTimestampArray(DateTime?[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); foreach (DateTime? val in vals) { @@ -585,7 +600,7 @@ public static unsafe void WriteString(string val, IBinaryStream stream) { int byteCnt = GetUtf8ByteCount(chars, charCnt); - stream.WriteInt(byteCnt); + WriteArrayLength(byteCnt, stream); stream.WriteString(chars, charCnt, byteCnt, Utf8); } @@ -775,7 +790,7 @@ public static string ReadString(IBinaryStream stream) */ public static void WriteStringArray(string[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); foreach (string val in vals) { @@ -789,6 +804,78 @@ public static void WriteStringArray(string[] vals, IBinaryStream stream) } } + /** + * + * Writes value of length of an array, which can be written in default format or varint encoding. + * Writing method depends on the constant . + * + * Varint encoding description. + * Integer value. + * Stream. + */ + public static void WriteArrayLength(int val, IBinaryStream stream) + { + if (!UseVarintArrayLenght) + stream.WriteInt(val); + else + WriteUvarint(val, stream); + } + + /** + * + * Reads value of length of an array, which can be presented in default format or varint encoding. + * Reading method depends on the constant . + * + * Varint encoding description. + * Stream. + * Integer value. + */ + public static int ReadArrayLength(IBinaryStream stream) + { + return !UseVarintArrayLenght ? stream.ReadInt() : ReadUvarint(stream); + } + + /** + * Writes an integer value in varint encoding. + * Varint encoding description. + * Integer value. + * Stream. + */ + public static void WriteUvarint(int val, IBinaryStream stream) + { + while ((val & 0xFFFFFF80) != 0) + { + stream.WriteByte((byte)((val & 0x7F) | 0x80)); + val = (int)((uint)val >> 7); + } + + stream.WriteByte((byte)(val & 0x7F)); + } + + /** + * Reads an integer value which is presented in varint encoding. + * Varint encoding description. + * Stream. + * Decoded integer value. + */ + public static int ReadUvarint(IBinaryStream stream) + { + int val = 0; + int bits = 0; + int b; + + while (((b = stream.ReadByte()) & 0x80) != 0) + { + val |= (int)((uint)(b & 0x7F) << bits); + bits += 7; + + if (bits > 35) + throw new BinaryObjectException("Varint reading failed, sequence length is too long"); + } + + return val | (int)((uint)b << bits); + } + /** * Write decimal value. * Decimal value. @@ -816,7 +903,7 @@ public static void WriteDecimal(decimal val, IBinaryStream stream) if (idx == -1) { // Writing zero. - stream.WriteInt(1); + WriteArrayLength(1, stream); stream.WriteByte(0); } else @@ -840,14 +927,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream) { if ((part24 & 0x80) == 0x80) { - stream.WriteInt(len + 1); + WriteArrayLength(len + 1, stream); stream.WriteByte((byte)(neg ? -0x80 : ByteZero)); neg = false; } else - stream.WriteInt(len); + WriteArrayLength(len, stream); stream.WriteByte((byte)(neg ? ((sbyte)part24 | -0x80) : part24)); stream.WriteByte((byte)part16); @@ -858,14 +945,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream) { if ((part16 & 0x80) == 0x80) { - stream.WriteInt(len); + WriteArrayLength(len, stream); stream.WriteByte((byte)(neg ? -0x80 : ByteZero)); neg = false; } else - stream.WriteInt(len - 1); + WriteArrayLength(len - 1, stream); stream.WriteByte((byte)(neg ? ((sbyte)part16 | -0x80) : part16)); stream.WriteByte((byte)part8); @@ -875,14 +962,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream) { if ((part8 & 0x80) == 0x80) { - stream.WriteInt(len - 1); + WriteArrayLength(len - 1, stream); stream.WriteByte((byte)(neg ? -0x80 : ByteZero)); neg = false; } else - stream.WriteInt(len - 2); + WriteArrayLength(len - 2, stream); stream.WriteByte((byte)(neg ? ((sbyte)part8 | -0x80) : part8)); stream.WriteByte((byte)part0); @@ -891,14 +978,14 @@ public static void WriteDecimal(decimal val, IBinaryStream stream) { if ((part0 & 0x80) == 0x80) { - stream.WriteInt(len - 2); + WriteArrayLength(len - 2, stream); stream.WriteByte((byte)(neg ? -0x80 : ByteZero)); neg = false; } else - stream.WriteInt(len - 3); + WriteArrayLength(len - 3, stream); stream.WriteByte((byte)(neg ? ((sbyte)part0 | -0x80) : part0)); } @@ -979,7 +1066,7 @@ public static void WriteDecimal(decimal val, IBinaryStream stream) */ public static void WriteDecimalArray(decimal?[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); foreach (var val in vals) { @@ -1001,7 +1088,7 @@ public static void WriteDecimalArray(decimal?[] vals, IBinaryStream stream) */ public static decimal?[] ReadDecimalArray(IBinaryStream stream) { - int len = stream.ReadInt(); + int len = ReadArrayLength(stream); var vals = new decimal?[len]; @@ -1141,7 +1228,7 @@ public static unsafe Guid ReadGuidSlow(IBinaryStream stream) /// Stream. public static void WriteGuidArray(Guid?[] vals, IBinaryStream stream) { - stream.WriteInt(vals.Length); + WriteArrayLength(vals.Length, stream); foreach (Guid? val in vals) { @@ -1187,7 +1274,7 @@ public static void WriteArray(Array val, BinaryWriter ctx, int? elemTypeId = nul ctx.WriteString(elemType.FullName); } - stream.WriteInt(val.Length); + WriteArrayLength(val.Length, stream); for (int i = 0; i < val.Length; i++) ctx.Write(val.GetValue(i)); @@ -1233,7 +1320,7 @@ public static T[] ReadArray(BinaryReader ctx, bool typed) ctx.ReadString(); } - int len = stream.ReadInt(); + int len = ReadArrayLength(stream); var vals = new T[len]; @@ -1252,7 +1339,7 @@ public static T[] ReadArray(BinaryReader ctx, bool typed) /// Timestamp array. public static DateTime?[] ReadTimestampArray(IBinaryStream stream) { - int len = stream.ReadInt(); + int len = ReadArrayLength(stream); DateTime?[] vals = new DateTime?[len]; @@ -1428,7 +1515,8 @@ public static IDictionary ReadDictionary(BinaryReader ctx, Func Date: Wed, 30 Aug 2017 16:16:06 +0300 Subject: [PATCH 2/4] IGNITE-5153 CPP: Introduced varint encoding in C++ --- .../include/ignite/binary/binary_raw_reader.h | 2 +- .../ignite/impl/binary/binary_reader_impl.h | 362 +++++------------- .../include/ignite/impl/binary/binary_utils.h | 93 ++++- .../ignite/impl/binary/binary_writer_impl.h | 112 +++--- .../interop/interop_stream_position_guard.h | 14 +- .../src/impl/binary/binary_reader_impl.cpp | 337 +++++++++++++--- .../binary/src/impl/binary/binary_utils.cpp | 180 ++++++++- .../src/impl/binary/binary_writer_impl.cpp | 132 ++++++- .../include/ignite/common/big_integer.h | 6 +- .../common/include/ignite/common/decimal.h | 199 +--------- .../include/ignite/common/fixed_size_array.h | 4 +- .../include/ignite/common/platform_utils.h | 18 +- .../cpp/common/include/ignite/guid.h | 10 + .../cpp/common/src/common/big_integer.cpp | 25 +- .../cpp/common/src/common/decimal.cpp | 206 +++++++++- modules/platforms/cpp/core-test/Makefile.am | 3 +- .../include/ignite/binary_test_utils.h | 20 +- .../core-test/include/ignite/complex_type.h | 19 + .../cpp/core-test/include/ignite/test_type.h | 21 + .../cpp/core-test/include/ignite/test_utils.h | 10 + .../cpp/core-test/src/binary_object_test.cpp | 8 +- .../src/binary_reader_writer_raw_test.cpp | 41 +- .../src/binary_reader_writer_test.cpp | 41 +- .../cpp/core-test/src/binary_utils_test.cpp | 291 ++++++++++++++ .../cpp/core-test/src/test_utils.cpp | 21 + .../cpp/odbc-test/src/column_test.cpp | 74 ++-- .../cpp/odbc/include/ignite/odbc/column.h | 4 +- .../cpp/odbc/include/ignite/odbc/utility.h | 16 - .../platforms/cpp/odbc/src/app/parameter.cpp | 2 +- modules/platforms/cpp/odbc/src/column.cpp | 12 +- modules/platforms/cpp/odbc/src/utility.cpp | 50 --- 31 files changed, 1508 insertions(+), 825 deletions(-) create mode 100644 modules/platforms/cpp/core-test/src/binary_utils_test.cpp diff --git a/modules/platforms/cpp/binary/include/ignite/binary/binary_raw_reader.h b/modules/platforms/cpp/binary/include/ignite/binary/binary_raw_reader.h index c06cb91edf848..144e610c7c13d 100644 --- a/modules/platforms/cpp/binary/include/ignite/binary/binary_raw_reader.h +++ b/modules/platforms/cpp/binary/include/ignite/binary/binary_raw_reader.h @@ -317,7 +317,7 @@ namespace ignite if (len != -1) { - ignite::common::FixedSizeArray arr(len + 1); + common::FixedSizeArray arr(len + 1); ReadString(arr.GetData(), static_cast(arr.GetSize())); diff --git a/modules/platforms/cpp/binary/include/ignite/impl/binary/binary_reader_impl.h b/modules/platforms/cpp/binary/include/ignite/impl/binary/binary_reader_impl.h index 4a0e2d43383f2..efd364ff5c9dd 100644 --- a/modules/platforms/cpp/binary/include/ignite/impl/binary/binary_reader_impl.h +++ b/modules/platforms/cpp/binary/include/ignite/impl/binary/binary_reader_impl.h @@ -22,18 +22,19 @@ #include #include +#include -#include "ignite/impl/interop/interop_input_stream.h" -#include "ignite/impl/binary/binary_common.h" -#include "ignite/impl/binary/binary_id_resolver.h" -#include "ignite/impl/binary/binary_schema.h" -#include "ignite/common/utils.h" -#include "ignite/binary/binary_consts.h" -#include "ignite/binary/binary_type.h" -#include "ignite/guid.h" -#include "ignite/date.h" -#include "ignite/timestamp.h" -#include "ignite/time.h" +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include namespace ignite { @@ -553,6 +554,21 @@ namespace ignite */ int32_t ReadTimeArray(const char* fieldName, Time* res, const int32_t len); + /** + * Read Decimal. Maps to "Decimal" type in Java. + * + * @param res Result. + */ + void ReadDecimal(common::Decimal& res); + + /** + * Read Decimal. Maps to "Decimal" type in Java. + * + * @param fieldName Field name. + * @param res Result. + */ + void ReadDecimal(const char* fieldName, common::Decimal& res); + /** * Read string. * @@ -930,77 +946,15 @@ namespace ignite case IGNITE_HDR_FULL: { - typedef ignite::binary::BinaryType BType; - - int8_t protoVer = stream->ReadInt8(); - - if (protoVer != IGNITE_PROTO_VER) { - IGNITE_ERROR_2(ignite::IgniteError::IGNITE_ERR_BINARY, - "Unsupported binary protocol version: ", protoVer); - } - - int16_t flags = stream->ReadInt16(); - - if (flags & IGNITE_BINARY_FLAG_COMPACT_FOOTER) { - IGNITE_ERROR_2(ignite::IgniteError::IGNITE_ERR_BINARY, - "Unsupported binary protocol flag: IGNITE_BINARY_FLAG_COMPACT_FOOTER: ", - IGNITE_BINARY_FLAG_COMPACT_FOOTER); - } - - int32_t typeId = stream->ReadInt32(); - int32_t hashCode = stream->ReadInt32(); - int32_t len = stream->ReadInt32(); - - // Ignoring Schema Id for now. - stream->ReadInt32(); - - int32_t schemaOrRawOff = stream->ReadInt32(); - - int32_t rawOff; - int32_t footerBegin; - - if (flags & IGNITE_BINARY_FLAG_HAS_SCHEMA) - footerBegin = pos + schemaOrRawOff; - else - footerBegin = pos + len; - - BinaryOffsetType::Type schemaType; - - if (flags & IGNITE_BINARY_FLAG_OFFSET_ONE_BYTE) - schemaType = BinaryOffsetType::ONE_BYTE; - else if (flags & IGNITE_BINARY_FLAG_OFFSET_TWO_BYTES) - schemaType = BinaryOffsetType::TWO_BYTES; - else - schemaType = BinaryOffsetType::FOUR_BYTES; - - int32_t footerEnd; - - if (flags & IGNITE_BINARY_FLAG_HAS_RAW && - flags & IGNITE_BINARY_FLAG_HAS_SCHEMA) - { - // 4 is the size of RawOffset field at the end of the packet. - footerEnd = pos + len - 4; - - rawOff = stream->ReadInt32(footerEnd); - } - else - { - footerEnd = pos + len; - - rawOff = schemaOrRawOff; - } - - bool usrType = (flags & IGNITE_BINARY_FLAG_USER_TYPE) != 0; - TemplatedBinaryIdResolver idRslvr; - BinaryReaderImpl readerImpl(stream, &idRslvr, pos, usrType, - typeId, hashCode, len, rawOff, - footerBegin, footerEnd, schemaType); - ignite::binary::BinaryReader reader(&readerImpl); + BinaryReaderImpl readerImpl(stream, &idRslvr, pos); BType::Read(reader, res); + ignite::binary::BinaryReader reader(&readerImpl); + + ignite::binary::BinaryType::Read(reader, val); - stream->Position(pos + len); + stream->Position(pos + readerImpl.len); return; } @@ -1013,19 +967,6 @@ namespace ignite } } - /** - * Get NULL value for the given type. - */ - template - T GetNull() const - { - T res; - - ignite::binary::BinaryType::GetNull(res); - - return res; - } - /** * Get underlying stream. * @@ -1142,61 +1083,24 @@ namespace ignite * @return Result. */ template - T ReadRaw(T(*func)(interop::InteropInputStream*)) - { - { - CheckRawMode(true); - CheckSingleMode(true); - - return func(stream); - } - } + T ReadRaw(T (*func)(interop::InteropInputStream*)); /** * Read single value. * * @param fieldName Field name. * @param func Function to be invoked on stream. - * @param epxHdr Expected header. + * @param expHdr Expected header. * @param dflt Default value returned if field is not found. * @return Result. */ template T Read( - const char* fieldName, - T(*func) (interop::InteropInputStream*), - const int8_t expHdr, + const char* fieldName, + T (*func)(interop::InteropInputStream*), + const int8_t expHdr, T dflt - ) - { - { - CheckRawMode(false); - CheckSingleMode(true); - - int32_t fieldId = idRslvr->GetFieldId(typeId, fieldName); - int32_t fieldPos = FindField(fieldId); - - if (fieldPos <= 0) - return dflt; - - stream->Position(fieldPos); - - int8_t typeId = stream->ReadInt8(); - - if (typeId == IGNITE_HDR_NULL) - return dflt; - - if (typeId != expHdr) - { - int32_t pos = stream->Position(); - - IGNITE_ERROR_FORMATTED_3(IgniteError::IGNITE_ERR_BINARY, "Invalid type ID", - "position", pos, "expected", static_cast(expHdr), "actual", static_cast(typeId)) - } - - return func(stream); - } - } + ); /** * Read array in raw mode. @@ -1211,17 +1115,9 @@ namespace ignite int32_t ReadRawArray( T* res, const int32_t len, - void(*func)(interop::InteropInputStream*, T* const, const int32_t), + void (*func)(interop::InteropInputStream*, T* const, const int32_t), const int8_t expHdr - ) - { - { - CheckRawMode(true); - CheckSingleMode(true); - - return ReadArrayInternal(res, len, stream, func, expHdr); - } - } + ); /** * Read array. @@ -1237,28 +1133,10 @@ namespace ignite int32_t ReadArray( const char* fieldName, T* res, - const int32_t len, - void(*func)(interop::InteropInputStream*, T* const, const int32_t), + const int32_t len, + void (*func)(interop::InteropInputStream*, T* const, const int32_t), const int8_t expHdr - ) - { - { - CheckRawMode(false); - CheckSingleMode(true); - - int32_t fieldId = idRslvr->GetFieldId(typeId, fieldName); - int32_t fieldPos = FindField(fieldId); - - if (fieldPos <= 0) - return -1; - - stream->Position(fieldPos); - - int32_t realLen = ReadArrayInternal(res, len, stream, func, expHdr); - - return realLen; - } - } + ); /** * Internal read array routine. @@ -1275,30 +1153,9 @@ namespace ignite T* res, const int32_t len, interop::InteropInputStream* stream, - void(*func)(interop::InteropInputStream*, T* const, const int32_t), + void (*func)(interop::InteropInputStream*, T* const, const int32_t), const int8_t expHdr - ) - { - { - int8_t hdr = stream->ReadInt8(); - - if (hdr == expHdr) - { - int32_t realLen = stream->ReadInt32(); - - if (realLen == 0 || (res && len >= realLen)) - func(stream, res, realLen); - else - stream->Position(stream->Position() - 5); - - return realLen; - } - else if (hdr != IGNITE_HDR_NULL) - ThrowOnInvalidHeader(stream->Position() - 1, expHdr, hdr); - - return -1; - } - } + ); /** * Read nullable value. @@ -1310,23 +1167,32 @@ namespace ignite template static T ReadNullable( interop::InteropInputStream* stream, - T(*func)(interop::InteropInputStream*), + T (*func)(interop::InteropInputStream*), const int8_t expHdr - ) - { - { - int8_t hdr = stream->ReadInt8(); + ); - if (hdr == expHdr) - return func(stream); - else if (hdr == IGNITE_HDR_NULL) - return T(); - else { - ThrowOnInvalidHeader(stream->Position() - 1, expHdr, hdr); + /** + * Read nullable value. + * + * @param stream Stream. + * @param func Function to be invoked on stream. + * @param expHdr Expected header. + */ + template + static void ReadNullable( + interop::InteropInputStream* stream, + void(*func)(interop::InteropInputStream*, T&), + const int8_t expHdr, + T& res + ); - return T(); - } - } + /** + * Get NULL value for the given type. + */ + template + static T GetNull() + { + return BinaryUtils::GetDefaultValue(); } /** @@ -1368,6 +1234,18 @@ namespace ignite */ void CheckSession(int32_t expSes) const; + /** + * Constructor. + * Init reader internal state from object header. + * Assuming that current stream position is at object header + * beginning. + * + * @param stream Interop stream. + * @param idRslvr Binary ID resolver. + * @param pos Object position in the stream. + */ + BinaryReaderImpl(interop::InteropInputStream* stream, BinaryIdResolver* idRslvr, int32_t pos); + /** * Throw an error due to invalid header. * @@ -1413,9 +1291,20 @@ namespace ignite * * @param expHdr Expected header. * @param func Function to be applied to the stream. + * @return Result. */ template - T ReadTopObject0(const int8_t expHdr, T (*func)(ignite::impl::interop::InteropInputStream*)); + T ReadTopObject0(const int8_t expHdr, T(*func)(interop::InteropInputStream*)); + + /** + * Read value. + * + * @param expHdr Expected header. + * @param func Function to be applied to the stream. + * @param res Result. + */ + template + void ReadTopObject0(const int8_t expHdr, void(*func)(interop::InteropInputStream*, T&), T& res); }; template<> @@ -1455,73 +1344,10 @@ namespace ignite void IGNITE_IMPORT_EXPORT BinaryReaderImpl::ReadTopObject0