From 615301895d33e8b0480fe24abc97f46a8fc1549a Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Sat, 13 Feb 2016 07:58:42 -0500 Subject: [PATCH 1/8] [fix] AVRO-1667 --- .../org/apache/avro/io/parsing/Symbol.java | 13 ++- .../apache/avro/io/parsing/SymbolTest.java | 79 +++++++++++++++++++ 2 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 lang/java/avro/src/test/java/org/apache/avro/io/parsing/SymbolTest.java diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java index 08a9d14ec3f..4456fb074a5 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java @@ -131,8 +131,8 @@ static Symbol resolve(Symbol w, Symbol r) { } private static class Fixup { - public final Symbol[] symbols; - public final int pos; + public Symbol[] symbols; + public int pos; public Fixup(Symbol[] symbols, int pos) { this.symbols = symbols; @@ -190,6 +190,15 @@ static void flatten(Symbol[] in, int start, List l = map2.get(s); if (l == null) { System.arraycopy(p, 0, out, j, p.length); + // Fixups need to be relocated! + for (List value : map2.values()) { + for (Fixup fixup : value) { + if (fixup.symbols == p) { + fixup.symbols = out; + fixup.pos += j; + } + } + } } else { l.add(new Fixup(out, j)); } diff --git a/lang/java/avro/src/test/java/org/apache/avro/io/parsing/SymbolTest.java b/lang/java/avro/src/test/java/org/apache/avro/io/parsing/SymbolTest.java new file mode 100644 index 00000000000..c844a05b5fe --- /dev/null +++ b/lang/java/avro/src/test/java/org/apache/avro/io/parsing/SymbolTest.java @@ -0,0 +1,79 @@ +/* + * Copyright 2016 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.io.parsing; + + +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; +import junit.framework.Assert; +import org.apache.avro.Schema; +import org.junit.Test; + +/** + * Unit test for AVRO-1667. + * @author zoly + */ +public class SymbolTest { + + + private static final String SCHEMA = "{\"type\":\"record\",\"name\":\"SampleNode\"," + + "\"namespace\":\"org.spf4j.ssdump2.avro\",\n" + +" \"fields\":[\n" + +" {\"name\":\"count\",\"type\":\"int\",\"default\":0},\n" + +" {\"name\":\"subNodes\",\"type\":\n" + +" {\"type\":\"array\",\"items\":{\n" + +" \"type\":\"record\",\"name\":\"SamplePair\",\n" + +" \"fields\":[\n" + +" {\"name\":\"method\",\"type\":\n" + +" {\"type\":\"record\",\"name\":\"Method\",\n" + +" \"fields\":[\n" + +" {\"name\":\"declaringClass\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},\n" + +" {\"name\":\"methodName\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}}\n" + +" ]}},\n" + +" {\"name\":\"node\",\"type\":\"SampleNode\"}]}}}]}"; + + + @Test + public void testSomeMethod() throws IOException { + + Schema schema = new Schema.Parser().parse(SCHEMA); + + Symbol root = Symbol.root(new ResolvingGrammarGenerator() + .generate(schema, schema, new HashMap())); + validateNonNull(root, new HashSet()); + } + + private static void validateNonNull(final Symbol symb, Set seen) { + if (seen.contains(symb)) { + return; + } else { + seen.add(symb); + } + if (symb.production != null) { + for (Symbol s : symb.production) { + if (s == null) { + Assert.fail("invalid parsing tree should not contain nulls"); + } + if (s.kind != Symbol.Kind.ROOT) { + validateNonNull(s, seen);; + } + } + } + } + +} \ No newline at end of file From f95c5e227127149851a9e743c07ded4a3d3f3d38 Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Tue, 23 Feb 2016 08:58:27 -0500 Subject: [PATCH 2/8] [fix] AVRO-1799 --- .../avro/src/main/java/org/apache/avro/generic/GenericData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java index 3bdfc7a1e7d..594b12e26e9 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java +++ b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java @@ -546,7 +546,7 @@ protected void toString(Object datum, StringBuilder buffer) { buffer.append("\""); } else if (isBytes(datum)) { buffer.append("{\"bytes\": \""); - ByteBuffer bytes = (ByteBuffer)datum; + ByteBuffer bytes = ((ByteBuffer)datum).duplicate(); for (int i = bytes.position(); i < bytes.limit(); i++) buffer.append((char)bytes.get(i)); buffer.append("\"}"); From 7ef4bbdecbff4a57b08945eccc906f2173049321 Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Thu, 10 Mar 2016 14:56:35 -0500 Subject: [PATCH 3/8] [fix] AVRO-1667 improve unit test --- .../apache/avro/io/parsing/SymbolTest.java | 156 ++++++++++++++++-- 1 file changed, 146 insertions(+), 10 deletions(-) diff --git a/lang/java/avro/src/test/java/org/apache/avro/io/parsing/SymbolTest.java b/lang/java/avro/src/test/java/org/apache/avro/io/parsing/SymbolTest.java index c844a05b5fe..0a0bd6c7ff8 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/io/parsing/SymbolTest.java +++ b/lang/java/avro/src/test/java/org/apache/avro/io/parsing/SymbolTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 The Apache Software Foundation. + * Copyright 2015 The Apache Software Foundation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,19 +15,26 @@ */ package org.apache.avro.io.parsing; - +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Set; import junit.framework.Assert; import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.EncoderFactory; import org.junit.Test; -/** - * Unit test for AVRO-1667. - * @author zoly - */ + public class SymbolTest { @@ -35,6 +42,7 @@ public class SymbolTest { + "\"namespace\":\"org.spf4j.ssdump2.avro\",\n" + " \"fields\":[\n" + " {\"name\":\"count\",\"type\":\"int\",\"default\":0},\n" + +" {\"name\":\"otherNode\",\"type\":[\"null\",\"SampleNode\"], \"default\" : null},\n" + " {\"name\":\"subNodes\",\"type\":\n" + " {\"type\":\"array\",\"items\":{\n" + " \"type\":\"record\",\"name\":\"SamplePair\",\n" + @@ -45,19 +53,147 @@ public class SymbolTest { " {\"name\":\"declaringClass\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},\n" + " {\"name\":\"methodName\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}}\n" + " ]}},\n" + -" {\"name\":\"node\",\"type\":\"SampleNode\"}]}}}]}"; +" {\"name\":\"node\",\"type\":\"SampleNode\"}," + + "{\"name\":\"otherNode\",\"type\":\"SampleNode\"}]}}}" + + ",{\"name\":\"subNodes2\",\"type\":\n" + +" {\"type\":\"array\",\"items\": \"SampleNode\"}}" + + ",{\"name\":\"subNodes3\",\"type\":\n" + +" {\"type\":\"array\",\"items\": \"SampleNode\"}} ]}"; @Test - public void testSomeMethod() throws IOException { - + public void testValidSymbolTree() throws IOException { Schema schema = new Schema.Parser().parse(SCHEMA); Symbol root = Symbol.root(new ResolvingGrammarGenerator() .generate(schema, schema, new HashMap())); validateNonNull(root, new HashSet()); + + Schema samplePairSchema = schema.getField("subNodes").schema().getElementType(); + Schema methodSchema = samplePairSchema.getField("method").schema(); + + GenericData.Record method = new GenericData.Record(methodSchema); + method.put("methodName", "m1"); + method.put("declaringClass", "c1"); + + GenericData.Record samplePair = new GenericData.Record(samplePairSchema); + samplePair.put("method", method); + + GenericData.Record sampleNode1 = new GenericData.Record(schema); + sampleNode1.put("subNodes", Collections.EMPTY_LIST); + sampleNode1.put("subNodes2", Collections.EMPTY_LIST); + sampleNode1.put("subNodes3", Collections.EMPTY_LIST); + sampleNode1.put("count", 0); + sampleNode1.put("otherNode", null); + + samplePair.put("node", sampleNode1); + samplePair.put("otherNode", sampleNode1); + + GenericData.Record sampleNode2 = new GenericData.Record(schema); + sampleNode2.put("subNodes", Arrays.asList(samplePair)); + sampleNode2.put("subNodes2", Arrays.asList(sampleNode1)); + sampleNode2.put("subNodes3", Arrays.asList(sampleNode1)); + sampleNode2.put("count", 0); + sampleNode2.put("otherNode", null); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + GenericDatumWriter writer = new GenericDatumWriter(schema); + BinaryEncoder directBinaryEncoder = new EncoderFactory().directBinaryEncoder(bos, null); + writer.write(sampleNode2, directBinaryEncoder); + + ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); + GenericDatumReader reader = new GenericDatumReader(schema); + BinaryDecoder binaryDecoder = new DecoderFactory().binaryDecoder(bis, null); + Object read = reader.read(null, binaryDecoder); + + System.out.println(read); + Assert.assertEquals(read, sampleNode2); + + } + + +/** + * Test IDL: + * +@namespace("test") +protocol NodeTest { + + record Node { + + string id = ""; + + array nodes = []; + + } + + record NodePair { + Node node1; + Node node2; + + } + +} +*/ + + public static final Schema NODE_SCHEMA = new Schema.Parser().parse("{\"type\":\"record\",\"name\":" + + "\"Node\",\"namespace\":\"test\",\"doc\":\"test node\",\"fields\":[{\"name\":\"id\"," + + "\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"identifier\",\"default\":\"\"}," + + "{\"name\":\"nodes\",\"type\":{\"type\":\"array\",\"items\":{\"type\":\"record\",\"name\":\"NodePair\"," + + "\"doc\":\"test node pair\",\"fields\":[{\"name\":\"node1\",\"type\":\"Node\",\"doc\":\"node 1\"}," + + "{\"name\":\"node2\",\"type\":\"Node\",\"doc\":\"node 2\"}]}},\"doc\":\"sub nodes\",\"default\":[]}]}"); + public static final Schema NOPAIR_SCHEMA = new Schema.Parser().parse("{\"type\":\"record\",\"name\":\"NodePair\"," + + "\"namespace\":\"test\",\"doc\":\"test node pair\",\"fields\":[{\"name\":\"node1\"," + + "\"type\":{\"type\":\"record\",\"name\":\"Node\",\"doc\":\"test node\",\"fields\":[{\"name\":\"id\"," + + "\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"identifier\",\"default\":\"\"}," + + "{\"name\":\"nodes\",\"type\":{\"type\":\"array\",\"items\":\"NodePair\"},\"doc\":\"sub nodes\"," + + "\"default\":[]}]},\"doc\":\"node 1\"},{\"name\":\"node2\",\"type\":\"Node\",\"doc\":\"node 2\"}]}"); + + + @Test + public void testValidSymbolTree2() throws IOException { + + Symbol root = Symbol.root(new ResolvingGrammarGenerator() + .generate(NODE_SCHEMA, NODE_SCHEMA, new HashMap())); + validateNonNull(root, new HashSet()); + root = Symbol.root(new ResolvingGrammarGenerator() + .generate(NOPAIR_SCHEMA, NOPAIR_SCHEMA, new HashMap())); + validateNonNull(root, new HashSet()); + + GenericData.Record node1 = new GenericData.Record(NODE_SCHEMA); + node1.put("id", "1"); + node1.put("nodes", Collections.EMPTY_LIST); + + GenericData.Record node2 = new GenericData.Record(NODE_SCHEMA); + node2.put("id", "1"); + node2.put("nodes", Collections.EMPTY_LIST); + + + GenericData.Record pair = new GenericData.Record(NOPAIR_SCHEMA); + pair.put("node1", node1); + pair.put("node2", node2); + + GenericData.Record node = new GenericData.Record(NODE_SCHEMA); + node.put("id", "Root"); + node.put("nodes", Arrays.asList(pair)); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + GenericDatumWriter writer = new GenericDatumWriter(NODE_SCHEMA); + BinaryEncoder directBinaryEncoder = new EncoderFactory().directBinaryEncoder(bos, null); + writer.write(node, directBinaryEncoder); + + ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); + GenericDatumReader reader = new GenericDatumReader(NODE_SCHEMA); + BinaryDecoder binaryDecoder = new DecoderFactory().binaryDecoder(bis, null); + Object read = reader.read(null, binaryDecoder); + + System.out.println(read); + Assert.assertEquals(read, node); + } + + + private static void validateNonNull(final Symbol symb, Set seen) { if (seen.contains(symb)) { return; @@ -76,4 +212,4 @@ private static void validateNonNull(final Symbol symb, Set seen) { } } -} \ No newline at end of file +} From 64f9d2e4f66510509f4fc8df0edd5006ef3f48b7 Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Thu, 10 Mar 2016 14:59:49 -0500 Subject: [PATCH 4/8] [fix] AVRO-1667 implement Ryan's suggestion --- .../org/apache/avro/io/parsing/Symbol.java | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java index 4456fb074a5..61df847190b 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java @@ -131,8 +131,8 @@ static Symbol resolve(Symbol w, Symbol r) { } private static class Fixup { - public Symbol[] symbols; - public int pos; + private final Symbol[] symbols; + private final int pos; public Fixup(Symbol[] symbols, int pos) { this.symbols = symbols; @@ -191,13 +191,8 @@ static void flatten(Symbol[] in, int start, if (l == null) { System.arraycopy(p, 0, out, j, p.length); // Fixups need to be relocated! - for (List value : map2.values()) { - for (Fixup fixup : value) { - if (fixup.symbols == p) { - fixup.symbols = out; - fixup.pos += j; - } - } + for (List fixups : map2.values()) { + copyFixups(fixups, out, j, p); } } else { l.add(new Fixup(out, j)); @@ -209,6 +204,17 @@ static void flatten(Symbol[] in, int start, } } + private static void copyFixups(List fixups, Symbol[] out, int outPos, + Symbol[] toCopy) { + final int nrFixups = fixups.size(); + for (int i = 0; i < nrFixups; i++) { + Fixup fixup = fixups.get(i); + if (fixup.symbols == toCopy) { + fixups.add(new Fixup(out, fixup.pos + outPos)); + } + } + } + /** * Returns the amount of space required to flatten the given * sub-array of symbols. From d1a6e6223bf8c581507e789fbfc317dcaaf061c9 Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Thu, 10 Mar 2016 15:06:25 -0500 Subject: [PATCH 5/8] [revert] unintentional commit --- .../avro/src/main/java/org/apache/avro/generic/GenericData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java index 594b12e26e9..3bdfc7a1e7d 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java +++ b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java @@ -546,7 +546,7 @@ protected void toString(Object datum, StringBuilder buffer) { buffer.append("\""); } else if (isBytes(datum)) { buffer.append("{\"bytes\": \""); - ByteBuffer bytes = ((ByteBuffer)datum).duplicate(); + ByteBuffer bytes = (ByteBuffer)datum; for (int i = bytes.position(); i < bytes.limit(); i++) buffer.append((char)bytes.get(i)); buffer.append("\"}"); From b66cc35f9d80424a1d214954d0ff2784ba4d2450 Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Sat, 13 Feb 2016 07:58:42 -0500 Subject: [PATCH 6/8] [fix] AVRO-1667 --- .../org/apache/avro/io/parsing/Symbol.java | 13 ++- .../apache/avro/io/parsing/SymbolTest.java | 79 +++++++++++++++++++ 2 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 lang/java/avro/src/test/java/org/apache/avro/io/parsing/SymbolTest.java diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java index 08a9d14ec3f..4456fb074a5 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java @@ -131,8 +131,8 @@ static Symbol resolve(Symbol w, Symbol r) { } private static class Fixup { - public final Symbol[] symbols; - public final int pos; + public Symbol[] symbols; + public int pos; public Fixup(Symbol[] symbols, int pos) { this.symbols = symbols; @@ -190,6 +190,15 @@ static void flatten(Symbol[] in, int start, List l = map2.get(s); if (l == null) { System.arraycopy(p, 0, out, j, p.length); + // Fixups need to be relocated! + for (List value : map2.values()) { + for (Fixup fixup : value) { + if (fixup.symbols == p) { + fixup.symbols = out; + fixup.pos += j; + } + } + } } else { l.add(new Fixup(out, j)); } diff --git a/lang/java/avro/src/test/java/org/apache/avro/io/parsing/SymbolTest.java b/lang/java/avro/src/test/java/org/apache/avro/io/parsing/SymbolTest.java new file mode 100644 index 00000000000..c844a05b5fe --- /dev/null +++ b/lang/java/avro/src/test/java/org/apache/avro/io/parsing/SymbolTest.java @@ -0,0 +1,79 @@ +/* + * Copyright 2016 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.io.parsing; + + +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; +import junit.framework.Assert; +import org.apache.avro.Schema; +import org.junit.Test; + +/** + * Unit test for AVRO-1667. + * @author zoly + */ +public class SymbolTest { + + + private static final String SCHEMA = "{\"type\":\"record\",\"name\":\"SampleNode\"," + + "\"namespace\":\"org.spf4j.ssdump2.avro\",\n" + +" \"fields\":[\n" + +" {\"name\":\"count\",\"type\":\"int\",\"default\":0},\n" + +" {\"name\":\"subNodes\",\"type\":\n" + +" {\"type\":\"array\",\"items\":{\n" + +" \"type\":\"record\",\"name\":\"SamplePair\",\n" + +" \"fields\":[\n" + +" {\"name\":\"method\",\"type\":\n" + +" {\"type\":\"record\",\"name\":\"Method\",\n" + +" \"fields\":[\n" + +" {\"name\":\"declaringClass\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},\n" + +" {\"name\":\"methodName\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}}\n" + +" ]}},\n" + +" {\"name\":\"node\",\"type\":\"SampleNode\"}]}}}]}"; + + + @Test + public void testSomeMethod() throws IOException { + + Schema schema = new Schema.Parser().parse(SCHEMA); + + Symbol root = Symbol.root(new ResolvingGrammarGenerator() + .generate(schema, schema, new HashMap())); + validateNonNull(root, new HashSet()); + } + + private static void validateNonNull(final Symbol symb, Set seen) { + if (seen.contains(symb)) { + return; + } else { + seen.add(symb); + } + if (symb.production != null) { + for (Symbol s : symb.production) { + if (s == null) { + Assert.fail("invalid parsing tree should not contain nulls"); + } + if (s.kind != Symbol.Kind.ROOT) { + validateNonNull(s, seen);; + } + } + } + } + +} \ No newline at end of file From d4b84193284cca41711a996c37abde737dab40b3 Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Thu, 10 Mar 2016 14:56:35 -0500 Subject: [PATCH 7/8] [fix] AVRO-1667 improve unit test --- .../apache/avro/io/parsing/SymbolTest.java | 156 ++++++++++++++++-- 1 file changed, 146 insertions(+), 10 deletions(-) diff --git a/lang/java/avro/src/test/java/org/apache/avro/io/parsing/SymbolTest.java b/lang/java/avro/src/test/java/org/apache/avro/io/parsing/SymbolTest.java index c844a05b5fe..0a0bd6c7ff8 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/io/parsing/SymbolTest.java +++ b/lang/java/avro/src/test/java/org/apache/avro/io/parsing/SymbolTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 The Apache Software Foundation. + * Copyright 2015 The Apache Software Foundation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,19 +15,26 @@ */ package org.apache.avro.io.parsing; - +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Set; import junit.framework.Assert; import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.EncoderFactory; import org.junit.Test; -/** - * Unit test for AVRO-1667. - * @author zoly - */ + public class SymbolTest { @@ -35,6 +42,7 @@ public class SymbolTest { + "\"namespace\":\"org.spf4j.ssdump2.avro\",\n" + " \"fields\":[\n" + " {\"name\":\"count\",\"type\":\"int\",\"default\":0},\n" + +" {\"name\":\"otherNode\",\"type\":[\"null\",\"SampleNode\"], \"default\" : null},\n" + " {\"name\":\"subNodes\",\"type\":\n" + " {\"type\":\"array\",\"items\":{\n" + " \"type\":\"record\",\"name\":\"SamplePair\",\n" + @@ -45,19 +53,147 @@ public class SymbolTest { " {\"name\":\"declaringClass\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},\n" + " {\"name\":\"methodName\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}}\n" + " ]}},\n" + -" {\"name\":\"node\",\"type\":\"SampleNode\"}]}}}]}"; +" {\"name\":\"node\",\"type\":\"SampleNode\"}," + + "{\"name\":\"otherNode\",\"type\":\"SampleNode\"}]}}}" + + ",{\"name\":\"subNodes2\",\"type\":\n" + +" {\"type\":\"array\",\"items\": \"SampleNode\"}}" + + ",{\"name\":\"subNodes3\",\"type\":\n" + +" {\"type\":\"array\",\"items\": \"SampleNode\"}} ]}"; @Test - public void testSomeMethod() throws IOException { - + public void testValidSymbolTree() throws IOException { Schema schema = new Schema.Parser().parse(SCHEMA); Symbol root = Symbol.root(new ResolvingGrammarGenerator() .generate(schema, schema, new HashMap())); validateNonNull(root, new HashSet()); + + Schema samplePairSchema = schema.getField("subNodes").schema().getElementType(); + Schema methodSchema = samplePairSchema.getField("method").schema(); + + GenericData.Record method = new GenericData.Record(methodSchema); + method.put("methodName", "m1"); + method.put("declaringClass", "c1"); + + GenericData.Record samplePair = new GenericData.Record(samplePairSchema); + samplePair.put("method", method); + + GenericData.Record sampleNode1 = new GenericData.Record(schema); + sampleNode1.put("subNodes", Collections.EMPTY_LIST); + sampleNode1.put("subNodes2", Collections.EMPTY_LIST); + sampleNode1.put("subNodes3", Collections.EMPTY_LIST); + sampleNode1.put("count", 0); + sampleNode1.put("otherNode", null); + + samplePair.put("node", sampleNode1); + samplePair.put("otherNode", sampleNode1); + + GenericData.Record sampleNode2 = new GenericData.Record(schema); + sampleNode2.put("subNodes", Arrays.asList(samplePair)); + sampleNode2.put("subNodes2", Arrays.asList(sampleNode1)); + sampleNode2.put("subNodes3", Arrays.asList(sampleNode1)); + sampleNode2.put("count", 0); + sampleNode2.put("otherNode", null); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + GenericDatumWriter writer = new GenericDatumWriter(schema); + BinaryEncoder directBinaryEncoder = new EncoderFactory().directBinaryEncoder(bos, null); + writer.write(sampleNode2, directBinaryEncoder); + + ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); + GenericDatumReader reader = new GenericDatumReader(schema); + BinaryDecoder binaryDecoder = new DecoderFactory().binaryDecoder(bis, null); + Object read = reader.read(null, binaryDecoder); + + System.out.println(read); + Assert.assertEquals(read, sampleNode2); + + } + + +/** + * Test IDL: + * +@namespace("test") +protocol NodeTest { + + record Node { + + string id = ""; + + array nodes = []; + + } + + record NodePair { + Node node1; + Node node2; + + } + +} +*/ + + public static final Schema NODE_SCHEMA = new Schema.Parser().parse("{\"type\":\"record\",\"name\":" + + "\"Node\",\"namespace\":\"test\",\"doc\":\"test node\",\"fields\":[{\"name\":\"id\"," + + "\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"identifier\",\"default\":\"\"}," + + "{\"name\":\"nodes\",\"type\":{\"type\":\"array\",\"items\":{\"type\":\"record\",\"name\":\"NodePair\"," + + "\"doc\":\"test node pair\",\"fields\":[{\"name\":\"node1\",\"type\":\"Node\",\"doc\":\"node 1\"}," + + "{\"name\":\"node2\",\"type\":\"Node\",\"doc\":\"node 2\"}]}},\"doc\":\"sub nodes\",\"default\":[]}]}"); + public static final Schema NOPAIR_SCHEMA = new Schema.Parser().parse("{\"type\":\"record\",\"name\":\"NodePair\"," + + "\"namespace\":\"test\",\"doc\":\"test node pair\",\"fields\":[{\"name\":\"node1\"," + + "\"type\":{\"type\":\"record\",\"name\":\"Node\",\"doc\":\"test node\",\"fields\":[{\"name\":\"id\"," + + "\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"identifier\",\"default\":\"\"}," + + "{\"name\":\"nodes\",\"type\":{\"type\":\"array\",\"items\":\"NodePair\"},\"doc\":\"sub nodes\"," + + "\"default\":[]}]},\"doc\":\"node 1\"},{\"name\":\"node2\",\"type\":\"Node\",\"doc\":\"node 2\"}]}"); + + + @Test + public void testValidSymbolTree2() throws IOException { + + Symbol root = Symbol.root(new ResolvingGrammarGenerator() + .generate(NODE_SCHEMA, NODE_SCHEMA, new HashMap())); + validateNonNull(root, new HashSet()); + root = Symbol.root(new ResolvingGrammarGenerator() + .generate(NOPAIR_SCHEMA, NOPAIR_SCHEMA, new HashMap())); + validateNonNull(root, new HashSet()); + + GenericData.Record node1 = new GenericData.Record(NODE_SCHEMA); + node1.put("id", "1"); + node1.put("nodes", Collections.EMPTY_LIST); + + GenericData.Record node2 = new GenericData.Record(NODE_SCHEMA); + node2.put("id", "1"); + node2.put("nodes", Collections.EMPTY_LIST); + + + GenericData.Record pair = new GenericData.Record(NOPAIR_SCHEMA); + pair.put("node1", node1); + pair.put("node2", node2); + + GenericData.Record node = new GenericData.Record(NODE_SCHEMA); + node.put("id", "Root"); + node.put("nodes", Arrays.asList(pair)); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + GenericDatumWriter writer = new GenericDatumWriter(NODE_SCHEMA); + BinaryEncoder directBinaryEncoder = new EncoderFactory().directBinaryEncoder(bos, null); + writer.write(node, directBinaryEncoder); + + ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); + GenericDatumReader reader = new GenericDatumReader(NODE_SCHEMA); + BinaryDecoder binaryDecoder = new DecoderFactory().binaryDecoder(bis, null); + Object read = reader.read(null, binaryDecoder); + + System.out.println(read); + Assert.assertEquals(read, node); + } + + + private static void validateNonNull(final Symbol symb, Set seen) { if (seen.contains(symb)) { return; @@ -76,4 +212,4 @@ private static void validateNonNull(final Symbol symb, Set seen) { } } -} \ No newline at end of file +} From da17084f559371fecbadd5badca38b074b2941f3 Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Thu, 10 Mar 2016 14:59:49 -0500 Subject: [PATCH 8/8] [fix] AVRO-1667 implement Ryan's suggestion --- .../org/apache/avro/io/parsing/Symbol.java | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java index 4456fb074a5..61df847190b 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java @@ -131,8 +131,8 @@ static Symbol resolve(Symbol w, Symbol r) { } private static class Fixup { - public Symbol[] symbols; - public int pos; + private final Symbol[] symbols; + private final int pos; public Fixup(Symbol[] symbols, int pos) { this.symbols = symbols; @@ -191,13 +191,8 @@ static void flatten(Symbol[] in, int start, if (l == null) { System.arraycopy(p, 0, out, j, p.length); // Fixups need to be relocated! - for (List value : map2.values()) { - for (Fixup fixup : value) { - if (fixup.symbols == p) { - fixup.symbols = out; - fixup.pos += j; - } - } + for (List fixups : map2.values()) { + copyFixups(fixups, out, j, p); } } else { l.add(new Fixup(out, j)); @@ -209,6 +204,17 @@ static void flatten(Symbol[] in, int start, } } + private static void copyFixups(List fixups, Symbol[] out, int outPos, + Symbol[] toCopy) { + final int nrFixups = fixups.size(); + for (int i = 0; i < nrFixups; i++) { + Fixup fixup = fixups.get(i); + if (fixup.symbols == toCopy) { + fixups.add(new Fixup(out, fixup.pos + outPos)); + } + } + } + /** * Returns the amount of space required to flatten the given * sub-array of symbols.