Permalink
Browse files

Implemented ION Element Count field - in arrays

  • Loading branch information...
1 parent 0eeb780 commit ce93a3ba4a269ed3e3f956246bc9ee3b093e5642 @jjenkov committed Jan 19, 2016
@@ -5,7 +5,7 @@
*/
public class IonFieldTypes {
- /* Field type constants */
+ /* Core type ID constants */
public static final int BYTES = 0; //a series of raw bytes
public static final int TINY = 1; // a number between 1 and 15, useful for booleans and small enums
public static final int INT_POS = 2;
@@ -28,25 +28,19 @@
/*
- The two reserved types will be used for either:
-
- UTF_8 - 0 to 15 bytes
- UTF_8 - 16 to 31 bytes
- DATETIME
- TIME
- DURATION
+ Extended type ID constants - can be from 0 to 255 - but we use only from 16 to 255 to avoid
+ numeric clashes with the core type IDs.
+ */
+ public static final int ELEMENT_COUNT = 16;
- Which one we choose will be determined after measuring a lot of data traffic going through VStack
+ /*
+ complex type id (not short version)
+ reference
+ UTC time
+ */
- Extended types:
- - DATETIME
- - TIME
- - DURATION
- Note that it does not make sense to represent UTF_8 (0 to 15 bytes) and UTF_8 (16 to 31 bytes) as extended
- types. The one byte these types save, the lose again with the type extension byte.
- */
@@ -36,6 +36,7 @@ public int read(byte[] source, int sourceOffset, Object destination) {
//read array field element count
int elementCountLeadByte = source[sourceOffset++];
int elementCountLength = elementCountLeadByte & 15;
+ sourceOffset++; //step over extended type byte... todo maybe validate it?
int elementCount = 0;
for(int i=0; i<elementCountLength; i++){
@@ -39,6 +39,8 @@ public int read(byte[] source, int sourceOffset, Object destination) {
//read array field element count
int elementCountLeadByte = source[sourceOffset++];
int elementCountLength = elementCountLeadByte & 15;
+ sourceOffset++; //step over extended type byte... todo maybe validate it?
+
int elementCount = 0;
for(int i=0; i<elementCountLength; i++){
@@ -39,6 +39,8 @@ public int read(byte[] source, int sourceOffset, Object destination) {
//read array field element count
int elementCountLeadByte = source[sourceOffset++];
int elementCountLength = elementCountLeadByte & 15;
+ sourceOffset++; //step over extended type byte... todo maybe validate it?
+
int elementCount = 0;
for(int i=0; i<elementCountLength; i++){
@@ -38,6 +38,8 @@ public int read(byte[] source, int sourceOffset, Object destination) {
//read array field element count
int elementCountLeadByte = source[sourceOffset++];
int elementCountLength = elementCountLeadByte & 15;
+ sourceOffset++; //step over extended type byte... todo maybe validate it?
+
int elementCount = 0;
for(int i=0; i<elementCountLength; i++){
@@ -1,6 +1,6 @@
-package com.jenkov.iap.ion;
+package com.jenkov.iap.ion.read;
-import com.jenkov.iap.ion.read.IIonFieldReader;
+import com.jenkov.iap.ion.IonFieldTypes;
import java.lang.reflect.Field;
@@ -38,6 +38,8 @@ public int read(byte[] source, int sourceOffset, Object destination) {
//read array field element count
int elementCountLeadByte = source[sourceOffset++];
int elementCountLength = elementCountLeadByte & 15;
+ sourceOffset++; //step over extended type byte... todo maybe validate it?
+
int elementCount = 0;
for(int i=0; i<elementCountLength; i++){
@@ -52,13 +54,13 @@ public int read(byte[] source, int sourceOffset, Object destination) {
for(int i=0; i<elementCount; i++){
int elementLeadByte = 255 & source[sourceOffset++];
int elementLength = elementLeadByte & 15;
- short elementValue = 0;
+ int elementValue = 0;
for(int j=0; j<elementLength; j++){
elementValue <<= 8;
elementValue |= 255 & source[sourceOffset++];
}
if( (elementLeadByte >> 4) == IonFieldTypes.INT_POS){
- values[i] = elementValue;
+ values[i] = (short) elementValue;
} else {
values[i] = (short) -elementValue;
}
@@ -23,6 +23,7 @@
public int fieldType = 0;
public int fieldLengthLength = 0;
public int fieldLength = 0;
+ public int fieldTypeExtended = 0;
private long[] intoIndexStack = null;
private int intoIndexStackIndex = 0;
@@ -86,7 +87,18 @@ public IonReader parse() {
break;
}
- //fine for all fields that use the lengthLength field normally - meaning non-compact fields.
+ case IonFieldTypes.EXTENDED : {
+ this.fieldTypeExtended = this.source[index++]; //read extended field type - first byte after lead byte
+ switch(this.fieldTypeExtended) {
+
+ case IonFieldTypes.ELEMENT_COUNT : {
+ this.fieldLength = this.fieldLengthLength;
+ }
+ }
+ break;
+ }
+
+ //fine for all fields that use the lengthLength field normally - meaning Normal length fields (not Short and Tiny).
default : {
this.fieldLength = 0;
for(int i=0; i<this.fieldLengthLength; i++){
@@ -377,6 +389,20 @@ public String readKeyShortAsUtf8String(){
return new String(this.source, this.index, this.fieldLength);
}
+ public long readElementCount() {
+ //this.index++; //move over the first byte after lead byte - because this byte is the extended field id byte.
+
+ int valueIndex = this.index;
+ long value = 255 & this.source[valueIndex++];
+ for(int i=1; i<this.fieldLengthLength; i++){
+ value <<= 8;
+ value |= 255 & this.source[valueIndex++];
+ }
+
+ return value;
+
+ }
+
public boolean matches(byte[] expected) {
if(expected.length != this.fieldLength){
return false;
@@ -57,7 +57,9 @@ public int writeValueField(Object sourceObject, byte[] dest, int destOffset, int
destOffset += arrayLengthLength;
//write element count
- dest[destOffset++] = (byte) (255 & ((IonFieldTypes.INT_POS << 4) | elementCountLengthLength) );
+ dest[destOffset++] = (byte) (255 & ((IonFieldTypes.EXTENDED << 4) | elementCountLengthLength) );
+ dest[destOffset++] = (byte) IonFieldTypes.ELEMENT_COUNT;
+
for(int i=(elementCountLengthLength-1)*8; i >= 0; i-=8){
dest[destOffset++] = (byte) (255 & (elementCount >> i));
}
@@ -57,7 +57,9 @@ public int writeValueField(Object sourceObject, byte[] dest, int destOffset, int
destOffset += arrayLengthLength;
//write element count
- dest[destOffset++] = (byte) (255 & ((IonFieldTypes.INT_POS << 4) | elementCountLengthLength) );
+ dest[destOffset++] = (byte) (255 & ((IonFieldTypes.EXTENDED << 4) | elementCountLengthLength) );
+ dest[destOffset++] = (byte) IonFieldTypes.ELEMENT_COUNT;
+
for(int i=(elementCountLengthLength-1)*8; i >= 0; i-=8){
dest[destOffset++] = (byte) (255 & (elementCount >> i));
}
@@ -57,7 +57,9 @@ public int writeValueField(Object sourceObject, byte[] dest, int destOffset, int
destOffset += arrayLengthLength;
//write element count
- dest[destOffset++] = (byte) (255 & ((IonFieldTypes.INT_POS << 4) | elementCountLengthLength) );
+ dest[destOffset++] = (byte) (255 & ((IonFieldTypes.EXTENDED << 4) | elementCountLengthLength) );
+ dest[destOffset++] = (byte) IonFieldTypes.ELEMENT_COUNT;
+
for(int i=(elementCountLengthLength-1)*8; i >= 0; i-=8){
dest[destOffset++] = (byte) (255 & (elementCount >> i));
}
@@ -57,7 +57,9 @@ public int writeValueField(Object sourceObject, byte[] dest, int destOffset, int
destOffset += arrayLengthLength;
//write element count
- dest[destOffset++] = (byte) (255 & ((IonFieldTypes.INT_POS << 4) | elementCountLengthLength) );
+ dest[destOffset++] = (byte) (255 & ((IonFieldTypes.EXTENDED << 4) | elementCountLengthLength) );
+ dest[destOffset++] = (byte) IonFieldTypes.ELEMENT_COUNT;
+
for(int i=(elementCountLengthLength-1)*8; i >= 0; i-=8){
dest[destOffset++] = (byte) (255 & (elementCount >> i));
}
@@ -57,7 +57,9 @@ public int writeValueField(Object sourceObject, byte[] dest, int destOffset, int
destOffset += arrayLengthLength;
//write element count
- dest[destOffset++] = (byte) (255 & ((IonFieldTypes.INT_POS << 4) | elementCountLengthLength) );
+ dest[destOffset++] = (byte) (255 & ((IonFieldTypes.EXTENDED << 4) | elementCountLengthLength) );
+ dest[destOffset++] = (byte) IonFieldTypes.ELEMENT_COUNT;
+
for(int i=(elementCountLengthLength-1)*8; i >= 0; i-=8){
dest[destOffset++] = (byte) (255 & (elementCount >> i));
}
@@ -24,6 +24,7 @@ public IonObjectWriter(Class typeClass) {
}
}
+
public int writeObject(Object src, int maxLengthLength, byte[] destination, int destinationOffset){
destination[destinationOffset++] = (byte) (255 & ((IonFieldTypes.OBJECT << 4) | maxLengthLength));
@@ -50,4 +51,6 @@ public int writeObject(Object src, int maxLengthLength, byte[] destination, int
}
+
+
}
@@ -403,6 +403,28 @@ public void writeComplexTypeIdShort(byte[] value){
this.destIndex += length;
}
+
+ /*
+ Extended field types
+ */
+ public void writeElementCount(long elementCount){
+ int lengthLength = IonUtil.lengthOfInt64Value(elementCount);
+ this.dest[this.destIndex++] = (byte) (255 & ((IonFieldTypes.EXTENDED << 4) | lengthLength));
+ this.dest[this.destIndex++] = IonFieldTypes.ELEMENT_COUNT; //extended type id follows after lead byte.
+
+ for(int i=(lengthLength-1)*8; i >= 0; i-=8){
+ this.dest[this.destIndex++] = (byte) (255 & (elementCount >> i));
+ }
+
+ }
+
+
+ /*
+ ======================================================
+ Static versions follow below, of same methods as above
+ ======================================================
+ */
+
public static int writeBytes(byte[] dest, int destOffset, byte[] value){
if(value == null){
@@ -795,5 +817,21 @@ public static int writeDirect(byte[] dest, int destOffset, byte[] ionFieldBytes)
}
+ /*
+ Extended field types
+ */
+ public static int writeElementCount(byte[] dest, int destOffset, long elementCount){
+ int lengthLength = IonUtil.lengthOfInt64Value(elementCount);
+ dest[destOffset++] = (byte) (255 & ((IonFieldTypes.EXTENDED << 4) | lengthLength));
+ dest[destOffset++] = IonFieldTypes.ELEMENT_COUNT; //extended type id follows after lead byte.
+
+ for(int i=(lengthLength-1)*8; i >= 0; i-=8){
+ dest[destOffset++] = (byte) (255 & (elementCount >> i));
+ }
+
+ return 2 + lengthLength; // 1 lead byte, 1 extended type id byte, lengthLength element count bytes
+ }
+
+
}
@@ -336,9 +336,47 @@ public void testReadKeyCompact() {
}
+ @Test
+ public void testReadElementCount() {
+ byte[] dest = new byte[10 * 1024];
+
+ int index = 0;
+ index += IonWriter.writeElementCount(dest, index, 1024);
+ index += IonWriter.writeElementCount(dest, index, 2048);
+
+ assertEquals(8, index);
+
+ reader.setSource(dest, 0, dest.length);
+ reader.next();
+ reader.parse();
+
+ assertEquals(IonFieldTypes.EXTENDED , reader.fieldType);
+ assertEquals(IonFieldTypes.ELEMENT_COUNT, reader.fieldTypeExtended);
+ assertEquals(2, reader.fieldLengthLength);
+ assertEquals(2, reader.fieldLength);
+
+ int offset = 2;
+ assertEquals(1024 >> 8 , 255 & dest[offset++]);
+ assertEquals(1024 & 255, 255 & dest[offset++]);
+
+ reader.next();
+ reader.parse();
+ assertEquals(IonFieldTypes.EXTENDED , reader.fieldType);
+ assertEquals(IonFieldTypes.ELEMENT_COUNT, reader.fieldTypeExtended);
+ assertEquals(2, reader.fieldLengthLength);
+ assertEquals(2, reader.fieldLength);
+
+ offset = 6;
+ assertEquals(2048 >> 8 , 255 & dest[offset++]);
+ assertEquals(2048 & 255, 255 & dest[offset++]);
+
+ }
+
+
+
@Test
- public void testParseIntoAndOutOf() {
+ public void testMoveIntoAndOutOf() {
byte[] source = new byte[10 * 1024];
int index = 0;
@@ -535,4 +573,6 @@ private Map parseObject() {
}
+
+
}
Oops, something went wrong.

0 comments on commit ce93a3b

Please sign in to comment.