From edabd46137f6757b324769647d9729f935f9c988 Mon Sep 17 00:00:00 2001 From: Paul Dudenkov Date: Mon, 4 Jul 2016 13:49:24 +0300 Subject: [PATCH 1/2] AVRO-1875 Added ability to encode and decode byte arrays in Base64. --- .../org/apache/avro/io/DecoderFactory.java | 38 ++++++++++ .../org/apache/avro/io/EncoderFactory.java | 73 +++++++++++++++++++ .../java/org/apache/avro/io/JsonDecoder.java | 21 +++++- .../java/org/apache/avro/io/JsonEncoder.java | 28 ++++++- .../java/org/apache/avro/io/TestEncoders.java | 24 ++++++ .../org/apache/avro/io/TestJsonDecoder.java | 14 ++++ 6 files changed, 195 insertions(+), 3 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/DecoderFactory.java b/lang/java/avro/src/main/java/org/apache/avro/io/DecoderFactory.java index 2e874ad8e82..7b9c7ac6001 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/DecoderFactory.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/DecoderFactory.java @@ -268,6 +268,44 @@ public JsonDecoder jsonDecoder(Schema schema, String input) return new JsonDecoder(schema, input); } + /** + * Creates a {@link JsonDecoder} using the InputStrim provided for reading + * data that conforms to the Schema provided. + *

+ * + * @param schema + * The Schema for data read from this JsonEncoder. Cannot be null. + * @param input + * The InputStream to read from. Cannot be null. + * @param decodeBase64 + * Decode byte arrays with Base64. + * @return A JsonEncoder configured with input and schema + * @throws IOException + */ + public JsonDecoder jsonDecoder(Schema schema, InputStream input, boolean decodeBase64) + throws IOException { + return new JsonDecoder(schema, input, decodeBase64); + } + + /** + * Creates a {@link JsonDecoder} using the String provided for reading data + * that conforms to the Schema provided. + *

+ * + * @param schema + * The Schema for data read from this JsonEncoder. Cannot be null. + * @param input + * The String to read from. Cannot be null. + * @param decodeBase64 + * Decode byte arrays with Base64. + * @return A JsonEncoder configured with input and schema + * @throws IOException + */ + public JsonDecoder jsonDecoder(Schema schema, String input, boolean decodeBase64) + throws IOException { + return new JsonDecoder(schema, input, decodeBase64); + } + /** * Creates a {@link ValidatingDecoder} wrapping the Decoder provided. This * ValidatingDecoder will ensure that operations against it conform to the diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/EncoderFactory.java b/lang/java/avro/src/main/java/org/apache/avro/io/EncoderFactory.java index 9ba1acc541d..d6e5d24d41c 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/EncoderFactory.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/EncoderFactory.java @@ -328,6 +328,79 @@ public JsonEncoder jsonEncoder(Schema schema, JsonGenerator gen) return new JsonEncoder(schema, gen); } + /** + * Creates a {@link JsonEncoder} using the OutputStream provided for writing + * data conforming to the Schema provided. + *

+ * {@link JsonEncoder} buffers its output. Data may not appear on the + * underlying OutputStream until {@link Encoder#flush()} is called. + *

+ * {@link JsonEncoder} is not thread-safe. + * + * @param encodeBase64 + * Encode byte arrays with Base64. + * @param schema + * The Schema for data written to this JsonEncoder. Cannot be null. + * @param out + * The OutputStream to write to. Cannot be null. + * @return A JsonEncoder configured with out and schema + * @throws IOException + */ + public JsonEncoder jsonEncoder(boolean encodeBase64, Schema schema, OutputStream out) + throws IOException { + return new JsonEncoder(encodeBase64, schema, out); + } + + /** + * Creates a {@link JsonEncoder} using the OutputStream provided for writing + * data conforming to the Schema provided with optional pretty printing. + *

+ * {@link JsonEncoder} buffers its output. Data may not appear on the + * underlying OutputStream until {@link Encoder#flush()} is called. + *

+ * {@link JsonEncoder} is not thread-safe. + * + * @param schema + * The Schema for data written to this JsonEncoder. Cannot be null. + * @param out + * The OutputStream to write to. Cannot be null. + * @param pretty + * Pretty print encoding. + * @param encodeBase64 + * Encode byte arrays with Base64. + * @return A JsonEncoder configured with out, schema and pretty + * @throws IOException + */ + public JsonEncoder jsonEncoder(Schema schema, OutputStream out, boolean pretty, boolean encodeBase64) + throws IOException { + return new JsonEncoder(schema, out, pretty, encodeBase64); + } + + /** + * Creates a {@link JsonEncoder} using the {@link JsonGenerator} provided for + * output of data conforming to the Schema provided. + *

+ * {@link JsonEncoder} buffers its output. Data may not appear on the + * underlying output until {@link Encoder#flush()} is called. + *

+ * {@link JsonEncoder} is not thread-safe. + * + * @param schema + * The Schema for data written to this JsonEncoder. Cannot be null. + * @param gen + * The JsonGenerator to write with. Cannot be null. + * @param encodeBase64 + * Encode byte arrays with Base64. + * @return A JsonEncoder configured with gen and schema + * @throws IOException + * @deprecated internal method + */ + @Deprecated + public JsonEncoder jsonEncoder(Schema schema, JsonGenerator gen, boolean encodeBase64) + throws IOException { + return new JsonEncoder(schema, gen, encodeBase64); + } + /** * Creates a {@link ValidatingEncoder} that wraps the Encoder provided. * This ValidatingEncoder will ensure that operations against it conform diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/JsonDecoder.java b/lang/java/avro/src/main/java/org/apache/avro/io/JsonDecoder.java index 34a1862de03..c380d21869b 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/JsonDecoder.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/JsonDecoder.java @@ -23,11 +23,12 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Stack; +import java.util.ArrayList; +import java.util.Base64; import org.apache.avro.AvroTypeException; import org.apache.avro.Schema; @@ -55,6 +56,7 @@ public class JsonDecoder extends ParsingDecoder private static JsonFactory jsonFactory = new JsonFactory(); Stack reorderBuffers = new Stack(); ReorderBuffer currentReorderBuffer; + private boolean decodeBase64 = false; private static class ReorderBuffer { public Map> savedFields = new HashMap>(); @@ -81,6 +83,16 @@ private JsonDecoder(Symbol root, String in) throws IOException { this(getSymbol(schema), in); } + JsonDecoder(Schema schema, InputStream in, boolean decodeBase64) throws IOException { + this(getSymbol(schema), in); + this.decodeBase64 = decodeBase64; + } + + JsonDecoder(Schema schema, String in, boolean decodeBase64) throws IOException { + this(getSymbol(schema), in); + this.decodeBase64 = decodeBase64; + } + private static Symbol getSymbol(Schema schema) { if (null == schema) { throw new NullPointerException("Schema cannot be null!"); @@ -261,7 +273,12 @@ public ByteBuffer readBytes(ByteBuffer old) throws IOException { } private byte[] readByteArray() throws IOException { - byte[] result = in.getText().getBytes(CHARSET); + byte[] result; + if (decodeBase64) { + result = Base64.getDecoder().decode(in.getText()); + } else { + result = in.getText().getBytes(CHARSET); + } return result; } diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/JsonEncoder.java b/lang/java/avro/src/main/java/org/apache/avro/io/JsonEncoder.java index 2c2b0c1d51b..055a1271359 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/JsonEncoder.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/JsonEncoder.java @@ -20,6 +20,8 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Base64; import java.util.BitSet; import org.apache.avro.AvroTypeException; @@ -47,6 +49,7 @@ public class JsonEncoder extends ParsingEncoder implements Parser.ActionHandler private static final String LINE_SEPARATOR = System.getProperty("line.separator"); final Parser parser; private JsonGenerator out; + private boolean encodeBase64 = false; /** * Has anything been written into the collections? */ @@ -66,6 +69,23 @@ public class JsonEncoder extends ParsingEncoder implements Parser.ActionHandler new Parser(new JsonGrammarGenerator().generate(sc), this); } + JsonEncoder(boolean encodeBase64, Schema sc, OutputStream out) throws IOException { + this(sc, getJsonGenerator(out, false)); + this.encodeBase64 = encodeBase64; + } + + JsonEncoder(Schema sc, OutputStream out, boolean pretty, boolean encodeBase64) throws IOException { + this(sc, getJsonGenerator(out, pretty)); + this.encodeBase64 = encodeBase64; + } + + JsonEncoder(Schema sc, JsonGenerator out, boolean encodeBase64) throws IOException { + configure(out); + this.parser = + new Parser(new JsonGrammarGenerator().generate(sc), this); + this.encodeBase64 = encodeBase64; + } + @Override public void flush() throws IOException { parser.processImplicitActions(); @@ -215,8 +235,14 @@ public void writeBytes(byte[] bytes, int start, int len) throws IOException { private void writeByteArray(byte[] bytes, int start, int len) throws IOException { - out.writeString( + if (encodeBase64) { + out.writeString( + Base64.getEncoder().encodeToString(Arrays.copyOfRange(bytes, start, len+start))); + } else { + out.writeString( new String(bytes, start, len, JsonDecoder.CHARSET)); + } + } @Override diff --git a/lang/java/avro/src/test/java/org/apache/avro/io/TestEncoders.java b/lang/java/avro/src/test/java/org/apache/avro/io/TestEncoders.java index 4d16f163505..ed3a2626770 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/io/TestEncoders.java +++ b/lang/java/avro/src/test/java/org/apache/avro/io/TestEncoders.java @@ -26,6 +26,7 @@ import org.apache.avro.Schema.Type; import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; import org.codehaus.jackson.JsonEncoding; import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.JsonGenerator; @@ -193,4 +194,27 @@ public void testJsonRecordOrderingWithProjection2() throws IOException { Assert.assertEquals("{\"a\": {\"a1\": null, \"a2\": true}}", o.toString()); } + @Test public void testJsonEncodingBase64() throws Exception { + String schemaSrc = "{\"type\":\"record\",\"name\":\"SampleConfiguration\",\"namespace\":\"org.kaaproject.kaa.demo.configuration\",\"fields\":[{\"name\":\"AddressList\",\"type\":[{\"type\":\"array\",\"items\":{\"type\":\"record\",\"name\":\"Link\",\"fields\":[{\"name\":\"label\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"displayName\":\"Site label\"},{\"name\":\"url\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"displayName\":\"Site URL\"},{\"name\":\"__uuid\",\"type\":[{\"type\":\"fixed\",\"name\":\"uuidT\",\"namespace\":\"org.kaaproject.configuration\",\"size\":16},\"null\"],\"displayName\":\"Record Id\",\"fieldAccess\":\"read_only\"}],\"displayName\":\"Site address\"}},\"null\"],\"displayName\":\"URLs list\"},{\"name\":\"__uuid\",\"type\":[\"org.kaaproject.configuration.uuidT\",\"null\"],\"displayName\":\"Record Id\",\"fieldAccess\":\"read_only\"}]}"; + Schema schema = new Schema.Parser().parse(schemaSrc); + //getting record to encode + DatumWriter datumWriter = new GenericDatumWriter(schema); + String recordWithLatin1 = "{\"AddressList\":{\"array\":[{\"label\":\"Kaa website\",\"url\":\"http://www.kaaproject.org\",\"__uuid\":{\"org.kaaproject.configuration.uuidT\":\"à\\u0001§ï\u0081yIã»g?ßòõ,\\u0017\"}},{\"label\":\"Kaa GitHub repository\",\"url\":\"https://github.com/kaaproject/kaa\",\"__uuid\":{\"org.kaaproject.configuration.uuidT\":\"}\\u001A\\u001A\u009E\\u0013£DÅ»\\be\\u0016We\\bð\"}},{\"label\":\"Kaa docs\",\"url\":\"http://docs.kaaproject.org/display/KAA/Kaa+IoT+Platform+Home\",\"__uuid\":{\"org.kaaproject.configuration.uuidT\":\"¢\\\\\u008Bæf»J\u0092\u0093\u0091\\u0019ÝŽJù\\\"\"}},{\"label\":\"Kaa configuration design reference\",\"url\":\"http://docs.kaaproject.org/display/KAA/Configuration\",\"__uuid\":{\"org.kaaproject.configuration.uuidT\":\"!ÙpQ\\u0018iNy\u0095Sj\u0081§\u007F$\u0082\"}},{\"label\":\"Hello world\",\"url\":\"http://hello.world\",\"__uuid\":{\"org.kaaproject.configuration.uuidT\":\"8Ž\\bŸ\u008EÐO7\u0097\u008Aæv\u009E|Ž\u0099\"}}]},\"__uuid\":{\"org.kaaproject.configuration.uuidT\":\"Jç]\u008F:¶@\\u0002\u0086>n^ÿ\\u0010×Ù\"}}"; + JsonDecoder jsonDecoder = DecoderFactory.get().jsonDecoder(schema, recordWithLatin1); + DatumReader datumReader = new GenericDatumReader(schema); + GenericRecord record = datumReader.read(null, jsonDecoder); + //encoding + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Encoder jsonEncoder = EncoderFactory.get().jsonEncoder(true, schema, baos); + datumWriter.write(record, jsonEncoder); + jsonEncoder.flush(); + baos.flush(); + byte[] bytes = baos.toByteArray(); + String actual = new String(bytes); + + String expected = "{\"AddressList\":{\"array\":[{\"label\":\"Kaa website\",\"url\":\"http://www.kaaproject.org\",\"__uuid\":{\"org.kaaproject.configuration.uuidT\":\"4AGn74F5SeO7Zz/f8vUsFw==\"}},{\"label\":\"Kaa GitHub repository\",\"url\":\"https://github.com/kaaproject/kaa\",\"__uuid\":{\"org.kaaproject.configuration.uuidT\":\"fRoanhOjRMW7CGUWV2UI8A==\"}},{\"label\":\"Kaa docs\",\"url\":\"http://docs.kaaproject.org/display/KAA/Kaa+IoT+Platform+Home\",\"__uuid\":{\"org.kaaproject.configuration.uuidT\":\"olyL5ma7SpKTkRndP0r5Ig==\"}},{\"label\":\"Kaa configuration design reference\",\"url\":\"http://docs.kaaproject.org/display/KAA/Configuration\",\"__uuid\":{\"org.kaaproject.configuration.uuidT\":\"IdlwURhpTnmVU2qBp38kgg==\"}},{\"label\":\"Hello world\",\"url\":\"http://hello.world\",\"__uuid\":{\"org.kaaproject.configuration.uuidT\":\"OD8IP47QTzeXiuZ2nnw/mQ==\"}}]},\"__uuid\":{\"org.kaaproject.configuration.uuidT\":\"Suddjzq2QAKGPm5e/xDX2Q==\"}}"; + Assert.assertNotNull("Resulting string shouldn't be null", actual); + Assert.assertEquals("Actual result of encoding is not equal to expected", expected, actual); + } + } diff --git a/lang/java/avro/src/test/java/org/apache/avro/io/TestJsonDecoder.java b/lang/java/avro/src/test/java/org/apache/avro/io/TestJsonDecoder.java index 6ee6fcbf943..aee01e9cb3d 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/io/TestJsonDecoder.java +++ b/lang/java/avro/src/test/java/org/apache/avro/io/TestJsonDecoder.java @@ -77,4 +77,18 @@ private void checkNumeric(String type, Object value) throws Exception { Assert.assertEquals(200, in.readLong()); in.skipArray(); } + + @Test public void testDecodingBase64() throws Exception { + String schemaSrc = "{\"type\":\"record\",\"name\":\"SampleConfiguration\",\"namespace\":\"org.kaaproject.kaa.demo.configuration\",\"fields\":[{\"name\":\"AddressList\",\"type\":[{\"type\":\"array\",\"items\":{\"type\":\"record\",\"name\":\"Link\",\"fields\":[{\"name\":\"label\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"displayName\":\"Site label\"},{\"name\":\"url\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"displayName\":\"Site URL\"},{\"name\":\"__uuid\",\"type\":[{\"type\":\"fixed\",\"name\":\"uuidT\",\"namespace\":\"org.kaaproject.configuration\",\"size\":16},\"null\"],\"displayName\":\"Record Id\",\"fieldAccess\":\"read_only\"}],\"displayName\":\"Site address\"}},\"null\"],\"displayName\":\"URLs list\"},{\"name\":\"__uuid\",\"type\":[\"org.kaaproject.configuration.uuidT\",\"null\"],\"displayName\":\"Record Id\",\"fieldAccess\":\"read_only\"}]}"; + Schema schema = new Schema.Parser().parse(schemaSrc); + String encoded = "{\"AddressList\":{\"array\":[{\"label\":\"Kaa website\",\"url\":\"http://www.kaaproject.org\",\"__uuid\":{\"org.kaaproject.configuration.uuidT\":\"4AGn74F5SeO7Zz/f8vUsFw==\"}},{\"label\":\"Kaa GitHub repository\",\"url\":\"https://github.com/kaaproject/kaa\",\"__uuid\":{\"org.kaaproject.configuration.uuidT\":\"fRoanhOjRMW7CGUWV2UI8A==\"}},{\"label\":\"Kaa docs\",\"url\":\"http://docs.kaaproject.org/display/KAA/Kaa+IoT+Platform+Home\",\"__uuid\":{\"org.kaaproject.configuration.uuidT\":\"olyL5ma7SpKTkRndP0r5Ig==\"}},{\"label\":\"Kaa configuration design reference\",\"url\":\"http://docs.kaaproject.org/display/KAA/Configuration\",\"__uuid\":{\"org.kaaproject.configuration.uuidT\":\"IdlwURhpTnmVU2qBp38kgg==\"}},{\"label\":\"Hello world\",\"url\":\"http://hello.world\",\"__uuid\":{\"org.kaaproject.configuration.uuidT\":\"OD8IP47QTzeXiuZ2nnw/mQ==\"}}]},\"__uuid\":{\"org.kaaproject.configuration.uuidT\":\"Suddjzq2QAKGPm5e/xDX2Q==\"}}"; + boolean base64 = true; + JsonDecoder jsonDecoder = DecoderFactory.get().jsonDecoder(schema, encoded, base64); + DatumReader datumReader = new GenericDatumReader(schema); + GenericRecord record = datumReader.read(null, jsonDecoder); + String actual = record.toString(); + String expected = "{\"AddressList\": [{\"label\": \"Kaa website\", \"url\": \"http://www.kaaproject.org\", \"__uuid\": [-32, 1, -89, -17, -127, 121, 73, -29, -69, 103, 63, -33, -14, -11, 44, 23]}, {\"label\": \"Kaa GitHub repository\", \"url\": \"https://github.com/kaaproject/kaa\", \"__uuid\": [125, 26, 26, -98, 19, -93, 68, -59, -69, 8, 101, 22, 87, 101, 8, -16]}, {\"label\": \"Kaa docs\", \"url\": \"http://docs.kaaproject.org/display/KAA/Kaa+IoT+Platform+Home\", \"__uuid\": [-94, 92, -117, -26, 102, -69, 74, -110, -109, -111, 25, -35, 63, 74, -7, 34]}, {\"label\": \"Kaa configuration design reference\", \"url\": \"http://docs.kaaproject.org/display/KAA/Configuration\", \"__uuid\": [33, -39, 112, 81, 24, 105, 78, 121, -107, 83, 106, -127, -89, 127, 36, -126]}, {\"label\": \"Hello world\", \"url\": \"http://hello.world\", \"__uuid\": [56, 63, 8, 63, -114, -48, 79, 55, -105, -118, -26, 118, -98, 124, 63, -103]}], \"__uuid\": [74, -25, 93, -113, 58, -74, 64, 2, -122, 62, 110, 94, -1, 16, -41, -39]}"; + Assert.assertNotNull("Resulting string shouldn't be null", actual); + Assert.assertEquals("Actual result of encoding is not equal to expected", expected, actual); + } } From f94fb1a9e9bda31cafce144b63a211adbd2abd07 Mon Sep 17 00:00:00 2001 From: Paul Dudenkov Date: Wed, 6 Jul 2016 16:48:58 +0300 Subject: [PATCH 2/2] removed first boolean param --- .../org/apache/avro/io/EncoderFactory.java | 23 ------------------- .../java/org/apache/avro/io/JsonEncoder.java | 5 ---- 2 files changed, 28 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/EncoderFactory.java b/lang/java/avro/src/main/java/org/apache/avro/io/EncoderFactory.java index d6e5d24d41c..3d246237f46 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/EncoderFactory.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/EncoderFactory.java @@ -328,29 +328,6 @@ public JsonEncoder jsonEncoder(Schema schema, JsonGenerator gen) return new JsonEncoder(schema, gen); } - /** - * Creates a {@link JsonEncoder} using the OutputStream provided for writing - * data conforming to the Schema provided. - *

- * {@link JsonEncoder} buffers its output. Data may not appear on the - * underlying OutputStream until {@link Encoder#flush()} is called. - *

- * {@link JsonEncoder} is not thread-safe. - * - * @param encodeBase64 - * Encode byte arrays with Base64. - * @param schema - * The Schema for data written to this JsonEncoder. Cannot be null. - * @param out - * The OutputStream to write to. Cannot be null. - * @return A JsonEncoder configured with out and schema - * @throws IOException - */ - public JsonEncoder jsonEncoder(boolean encodeBase64, Schema schema, OutputStream out) - throws IOException { - return new JsonEncoder(encodeBase64, schema, out); - } - /** * Creates a {@link JsonEncoder} using the OutputStream provided for writing * data conforming to the Schema provided with optional pretty printing. diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/JsonEncoder.java b/lang/java/avro/src/main/java/org/apache/avro/io/JsonEncoder.java index 055a1271359..bd0812f3c2c 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/JsonEncoder.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/JsonEncoder.java @@ -69,11 +69,6 @@ public class JsonEncoder extends ParsingEncoder implements Parser.ActionHandler new Parser(new JsonGrammarGenerator().generate(sc), this); } - JsonEncoder(boolean encodeBase64, Schema sc, OutputStream out) throws IOException { - this(sc, getJsonGenerator(out, false)); - this.encodeBase64 = encodeBase64; - } - JsonEncoder(Schema sc, OutputStream out, boolean pretty, boolean encodeBase64) throws IOException { this(sc, getJsonGenerator(out, pretty)); this.encodeBase64 = encodeBase64;