From 1c5205ef3f1516fa5f3a323057139bffd5758182 Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Wed, 18 Nov 2015 17:44:50 -0500 Subject: [PATCH 01/22] [add] AVRO-1723 and AVRO-1667 --- .../org/apache/avro/io/parsing/Symbol.java | 97 ++++---- .../avro/compiler/idl/SchemaResolver.java | 229 ++++++++++++++++++ .../org/apache/avro/compiler/idl/idl.jj | 7 +- .../compiler/src/test/idl/input/cycle.avdl | 24 ++ .../compiler/src/test/idl/output/cycle.avpr | 55 +++++ .../apache/avro/compiler/idl/TestCycle.java | 105 ++++++++ lang/java/ipc/nb-configuration.xml | 18 ++ 7 files changed, 489 insertions(+), 46 deletions(-) create mode 100644 lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java create mode 100644 lang/java/compiler/src/test/idl/input/cycle.avdl create mode 100644 lang/java/compiler/src/test/idl/output/cycle.avpr create mode 100644 lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestCycle.java create mode 100644 lang/java/ipc/nb-configuration.xml diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java index 08a9d14ec3f..9d733817155 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java @@ -60,7 +60,7 @@ public enum Kind { * the symbols that forms the production for this symbol. The * sequence is in the reverse order of production. This is useful * for easy copying onto parsing stack. - * + * * Please note that this is a final. So the production for a symbol * should be known before that symbol is constructed. This requirement * cannot be met for those symbols which are recursive (e.g. a record that @@ -77,8 +77,8 @@ public enum Kind { protected Symbol(Kind kind) { this(kind, null); } - - + + protected Symbol(Kind kind, Symbol[] production) { this.production = production; this.kind = kind; @@ -120,7 +120,7 @@ static Symbol alt(Symbol[] symbols, String[] labels) { static Symbol error(String e) { return new ErrorAction(e); } - + /** * A convenience method to construct a ResolvingAction. * @param w The writer symbol @@ -129,32 +129,32 @@ static Symbol error(String e) { static Symbol resolve(Symbol w, Symbol r) { return new ResolvingAction(w, r); } - + private static class Fixup { - public final Symbol[] symbols; - public final int pos; - + public Symbol[] symbols; + public int pos; + public Fixup(Symbol[] symbols, int pos) { this.symbols = symbols; this.pos = pos; } } - + public Symbol flatten(Map map, Map> map2) { return this; } - + public int flattenedSize() { return 1; } - + /** * Flattens the given sub-array of symbols into an sub-array of symbols. Every * Sequence in the input are replaced by its production recursively. * Non-Sequence symbols, they internally have other symbols * those internal symbols also get flattened. - * + * * The algorithm does a few tricks to handle recursive symbol definitions. * In order to avoid infinite recursion with recursive symbols, we have a map * of Symbol->Symbol. Before fully constructing a flattened symbol for a @@ -168,7 +168,7 @@ public int flattenedSize() { * has not not be fully constructed yet, we copy a bunch of nulls. * Fix-up remembers all those null patches. The fix-ups gets finally * filled when we know the symbols to occupy those patches. - * + * * @param in The array of input symbols to flatten * @param start The position where the input sub-array starts. * @param out The output that receives the flattened list of symbols. The @@ -190,6 +190,15 @@ static void flatten(Symbol[] in, int start, List l = map2.get(s); if (l == null) { System.arraycopy(p, 0, out, j, p.length); + // Fixups need to be relocated! + for (List value : map2.values()) { + for (Fixup fixup : value) { + if (fixup.symbols == p) { + fixup.symbols = out; + fixup.pos += j; + } + } + } } else { l.add(new Fixup(out, j)); } @@ -232,7 +241,7 @@ public Terminal(String printName) { public static class ImplicitAction extends Symbol { /** - * Set to true if and only if this implicit action is + * Set to true if and only if this implicit action is * a trailing action. That is, it is an action that follows * real symbol. E.g {@link Symbol#DEFAULT_END_ACTION}. */ @@ -241,13 +250,13 @@ public static class ImplicitAction extends Symbol { private ImplicitAction() { this(false); } - + private ImplicitAction(boolean isTrailing) { super(Kind.IMPLICIT_ACTION); this.isTrailing = isTrailing; } } - + protected static class Root extends Symbol { private Root(Symbol... symbols) { super(Kind.ROOT, makeProduction(symbols)); @@ -262,7 +271,7 @@ private static Symbol[] makeProduction(Symbol[] symbols) { return result; } } - + protected static class Sequence extends Symbol implements Iterable { private Sequence(Symbol[] productions) { super(Kind.SEQUENCE, productions); @@ -271,19 +280,19 @@ private Sequence(Symbol[] productions) { public Symbol get(int index) { return production[index]; } - + public int size() { return production.length; } - + public Iterator iterator() { return new Iterator() { private int pos = production.length; - + public boolean hasNext() { return 0 < pos; } - + public Symbol next() { if (0 < pos) { return production[--pos]; @@ -291,7 +300,7 @@ public Symbol next() { throw new NoSuchElementException(); } } - + public void remove() { throw new UnsupportedOperationException(); } @@ -306,7 +315,7 @@ public Sequence flatten(Map map, map.put(this, result); List l = new ArrayList(); map2.put(result, l); - + flatten(production, 0, result.production, 0, map, map2); for (Fixup f : l) { @@ -326,19 +335,19 @@ public final int flattenedSize() { public static class Repeater extends Symbol { public final Symbol end; - + private Repeater(Symbol end, Symbol... sequenceToRepeat) { super(Kind.REPEATER, makeProduction(sequenceToRepeat)); this.end = end; production[0] = this; } - + private static Symbol[] makeProduction(Symbol[] p) { Symbol[] result = new Symbol[p.length + 1]; System.arraycopy(p, 0, result, 1, p.length); return result; } - + @Override public Repeater flatten(Map map, Map> map2) { @@ -349,9 +358,9 @@ public Repeater flatten(Map map, } } - + /** - * Returns true if the Parser contains any Error symbol, indicating that it may fail + * Returns true if the Parser contains any Error symbol, indicating that it may fail * for some inputs. */ public static boolean hasErrors(Symbol symbol) { @@ -374,7 +383,7 @@ public static boolean hasErrors(Symbol symbol) { throw new RuntimeException("unknown symbol kind: " + symbol.kind); } } - + private static boolean hasErrors(Symbol root, Symbol[] symbols) { if(null != symbols) { for(Symbol s: symbols) { @@ -388,7 +397,7 @@ private static boolean hasErrors(Symbol root, Symbol[] symbols) { } return false; } - + public static class Alternative extends Symbol { public final Symbol[] symbols; public final String[] labels; @@ -397,15 +406,15 @@ private Alternative(Symbol[] symbols, String[] labels) { this.symbols = symbols; this.labels = labels; } - + public Symbol getSymbol(int index) { return symbols[index]; } - + public String getLabel(int index) { return labels[index]; } - + public int size() { return symbols.length; } @@ -454,7 +463,7 @@ public static class IntCheckAction extends Symbol { public static EnumAdjustAction enumAdjustAction(int rsymCount, Object[] adj) { return new EnumAdjustAction(rsymCount, adj); } - + public static class EnumAdjustAction extends IntCheckAction { public final Object[] adjustments; @Deprecated public EnumAdjustAction(int rsymCount, Object[] adjustments) { @@ -478,7 +487,7 @@ private ResolvingAction(Symbol writer, Symbol reader) { this.writer = writer; this.reader = reader; } - + @Override public ResolvingAction flatten(Map map, Map> map2) { @@ -487,7 +496,7 @@ public ResolvingAction flatten(Map map, } } - + public static SkipAction skipAction(Symbol symToSkip) { return new SkipAction(symToSkip); } @@ -498,7 +507,7 @@ public static class SkipAction extends ImplicitAction { super(true); this.symToSkip = symToSkip; } - + @Override public SkipAction flatten(Map map, Map> map2) { @@ -510,7 +519,7 @@ public SkipAction flatten(Map map, public static FieldAdjustAction fieldAdjustAction(int rindex, String fname) { return new FieldAdjustAction(rindex, fname); } - + public static class FieldAdjustAction extends ImplicitAction { public final int rindex; public final String fname; @@ -519,7 +528,7 @@ public static class FieldAdjustAction extends ImplicitAction { this.fname = fname; } } - + public static FieldOrderAction fieldOrderAction(Schema.Field[] fields) { return new FieldOrderAction(fields); } @@ -553,13 +562,13 @@ public static class UnionAdjustAction extends ImplicitAction { this.rindex = rindex; this.symToParse = symToParse; } - + @Override public UnionAdjustAction flatten(Map map, Map> map2) { return new UnionAdjustAction(rindex, symToParse.flatten(map, map2)); } - + } /** For JSON. */ @@ -573,11 +582,11 @@ public static class EnumLabelsAction extends IntCheckAction { super(symbols.size()); this.symbols = symbols; } - + public String getLabel(int n) { return symbols.get(n); } - + public int findLabel(String l) { if (l != null) { for (int i = 0; i < symbols.size(); i++) { @@ -619,7 +628,7 @@ public int findLabel(String l) { public static final Symbol RECORD_END = new ImplicitAction(true); public static final Symbol UNION_END = new ImplicitAction(true); public static final Symbol FIELD_END = new ImplicitAction(true); - + public static final Symbol DEFAULT_END_ACTION = new ImplicitAction(true); public static final Symbol MAP_KEY_MARKER = new Symbol.Terminal("map-key-marker"); diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java new file mode 100644 index 00000000000..df0c7ac2ef5 --- /dev/null +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java @@ -0,0 +1,229 @@ +/* + * Copyright 2015 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.compiler.idl; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.avro.JsonProperties; +import org.apache.avro.LogicalType; +import org.apache.avro.Protocol; +import org.apache.avro.Schema; +import org.codehaus.jackson.JsonNode; + +/** + * + * @author zoly + */ +final class SchemaResolver { + + private SchemaResolver() { } + + private static final String UR_SCHEMA_ATTR = "org.apache.avro.compiler.idl.unresolved.name"; + + static Schema unresolvedSchema(final String name) { + Schema schema = Schema.createRecord("UnresolvedSchema", "unresolved schema", + "org.apache.avro.compiler", false, Collections.EMPTY_LIST); + schema.addProp(UR_SCHEMA_ATTR, name); + return schema; + } + + static boolean isUnresolvedSchema(final Schema schema) { + return (schema.getType() == Schema.Type.RECORD && schema.getProp(UR_SCHEMA_ATTR) != null); + } + + static String getUnresolvedSchemaName(final Schema schema) { + String name = schema.getProp(UR_SCHEMA_ATTR); + if (name == null) { + throw new IllegalArgumentException("Schema " + schema + " is not a unresolved schema"); + } else { + return name; + } + } + + + + static Protocol resolve(final Protocol protocol) { + Protocol result = new Protocol(protocol.getName(), protocol.getDoc(), protocol.getNamespace()); + final Collection types = protocol.getTypes(); + List newSchemas = new ArrayList(types.size()); + Map processed = new HashMap(); + for (Schema schema : types) { + newSchemas.add(resolve(schema, protocol, processed)); + } + result.setTypes(newSchemas); + + for (Map.Entry entry : protocol.getMessages().entrySet()) { + Protocol.Message value = entry.getValue(); + Protocol.Message nvalue; + if (value.isOneWay()) { + Schema request = value.getRequest(); + nvalue = result.createMessage(value.getName(), value.getDoc(), + value.getObjectProps(), intern(request, processed)); + } else { + Schema request = value.getRequest(); + Schema response = value.getResponse(); + Schema errors = value.getErrors(); + nvalue = result.createMessage(value.getName(), value.getDoc(), + value.getObjectProps(), intern(request, processed), + intern(response, processed), intern(errors, processed)); + } + result.getMessages().put(entry.getKey(), nvalue); + } + copyProps(protocol, result); + return result; + } + + private static void copyProps(final JsonProperties from, final JsonProperties to) { + for (Map.Entry entry : from.getJsonProps().entrySet()) { + to.addProp(entry.getKey(), entry.getValue()); + } + } + + + static Schema resolve(final Schema schema, final Protocol protocol, final Map processed) { + final String fullName = schema.getFullName(); + if (fullName != null && processed.containsKey(fullName)) { + return processed.get(schema.getFullName()); + } else if (isUnresolvedSchema(schema)) { + final String unresolvedSchemaName = getUnresolvedSchemaName(schema); + Schema type = protocol.getType(unresolvedSchemaName); + if (type == null) { + throw new IllegalArgumentException("Cannot resolve " + unresolvedSchemaName); + } + return resolve(type, protocol, processed); + } else { + switch (schema.getType()) { + case RECORD: + Schema createRecord = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), + schema.isError()); + processed.put(schema.getFullName(), createRecord); + final List currFields = schema.getFields(); + List newFields = new ArrayList(currFields.size()); + for (Schema.Field field : currFields) { + Schema.Field nf = new Schema.Field(field.name(), resolve(field.schema(), protocol, processed), + field.doc(), field.defaultVal(), field.order()); + for (String alias : field.aliases()) { + nf.addAlias(alias); + } + newFields.add(nf); + } + createRecord.setFields(newFields); + final LogicalType lt = schema.getLogicalType(); + if (lt != null) { + lt.addToSchema(createRecord); + } + copyProps(schema, createRecord); + return createRecord; + case MAP: + Schema result = Schema.createMap(resolve(schema.getValueType(), protocol, processed)); + copyProps(schema, result); + return result; + case ARRAY: + Schema aresult = Schema.createArray(resolve(schema.getElementType(), protocol, processed)); + copyProps(schema, aresult); + return aresult; + case UNION: + final List uTypes = schema.getTypes(); + List newTypes = new ArrayList(uTypes.size()); + for (Schema s : uTypes) { + newTypes.add(resolve(s, protocol, processed)); + } + Schema bresult = Schema.createUnion(newTypes); + copyProps(schema, bresult); + return bresult; + case ENUM: + case FIXED: + case STRING: + case BYTES: + case INT: + case LONG: + case FLOAT: + case DOUBLE: + case BOOLEAN: + case NULL: + return schema; + default: + throw new RuntimeException("Unknown type: " + schema); + } + } + } + + public static Schema intern(final Schema schema, final Map processed) { + if (schema == null) { + return null; + } + final String fullName = schema.getFullName(); + if (fullName != null && processed.containsKey(fullName)) { + return processed.get(schema.getFullName()); + } else { + switch (schema.getType()) { + case RECORD: + Schema createRecord = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), + schema.isError()); + processed.put(schema.getFullName(), createRecord); + final List currFields = schema.getFields(); + List newFields = new ArrayList(currFields.size()); + for (Schema.Field field : currFields) { + Schema.Field nf = new Schema.Field(field.name(), intern(field.schema(), processed), + field.doc(), field.defaultVal(), field.order()); + for (String alias : field.aliases()) { + nf.addAlias(alias); + } + newFields.add(nf); + } + createRecord.setFields(newFields); + final LogicalType lt = schema.getLogicalType(); + if (lt != null) { + lt.addToSchema(createRecord); + } + copyProps(schema, createRecord); + return createRecord; + case MAP: + return Schema.createMap(intern(schema.getValueType(), processed)); + case ARRAY: + return Schema.createArray(intern(schema.getElementType(), processed)); + case UNION: + final List uTypes = schema.getTypes(); + List newTypes = new ArrayList(uTypes.size()); + for (Schema s : uTypes) { + newTypes.add(intern(s, processed)); + } + return Schema.createUnion(newTypes); + case ENUM: + case FIXED: + case STRING: + case BYTES: + case INT: + case LONG: + case FLOAT: + case DOUBLE: + case BOOLEAN: + case NULL: + return schema; + default: + throw new RuntimeException("Unknown type: " + schema); + } + } + } + + + + +} diff --git a/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj b/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj index 8f60b83b90b..a8f06234625 100644 --- a/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj +++ b/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj @@ -999,7 +999,7 @@ Protocol CompilationUnit(): ( < "\u001a" > )? ( )? - { return p; } + { return SchemaResolver.resolve(p); } } /* @@ -1465,7 +1465,10 @@ Schema ReferenceType(): name = namespace + "." + name; Schema type = names.get(name); if (type == null) - throw error("Undefined name '" + name + "'", token); + //throw error("Undefined name '" + name + "'", token); + { + type = SchemaResolver.unresolvedSchema(name); + } return type; } } diff --git a/lang/java/compiler/src/test/idl/input/cycle.avdl b/lang/java/compiler/src/test/idl/input/cycle.avdl new file mode 100644 index 00000000000..f4344316024 --- /dev/null +++ b/lang/java/compiler/src/test/idl/input/cycle.avdl @@ -0,0 +1,24 @@ +@namespace("org.apache.avro.gen") +protocol Cycle { + + record SampleNode { + int count = 0; + array subNodes; + } + + record Method { + string declaringClass; + string methodName; + } + + record SamplePair { + Method method; + SampleNode node; + } + + record SelfRef { + string something; + array subNodes = []; + } + +} diff --git a/lang/java/compiler/src/test/idl/output/cycle.avpr b/lang/java/compiler/src/test/idl/output/cycle.avpr new file mode 100644 index 00000000000..53658afd00e --- /dev/null +++ b/lang/java/compiler/src/test/idl/output/cycle.avpr @@ -0,0 +1,55 @@ +{ + "protocol" : "Cycle", + "namespace" : "org.apache.avro.gen", + "types" : [ { + "type" : "record", + "name" : "SampleNode", + "fields" : [ { + "name" : "count", + "type" : "int", + "default" : 0 + }, { + "name" : "subNodes", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "SamplePair", + "fields" : [ { + "name" : "method", + "type" : { + "type" : "record", + "name" : "Method", + "fields" : [ { + "name" : "declaringClass", + "type" : "string" + }, { + "name" : "methodName", + "type" : "string" + } ] + } + }, { + "name" : "node", + "type" : "SampleNode" + } ] + } + } + } ] + }, { + "type" : "record", + "name" : "SelfRef", + "fields" : [ { + "name" : "something", + "type" : "string" + }, { + "name" : "subNodes", + "type" : { + "type" : "array", + "items" : "SelfRef" + }, + "default" : [ ] + } ] + } ], + "messages" : { } +} + diff --git a/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestCycle.java b/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestCycle.java new file mode 100644 index 00000000000..a38e978ef7d --- /dev/null +++ b/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestCycle.java @@ -0,0 +1,105 @@ +/* + * Copyright 2015 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.compiler.idl; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import junit.framework.Assert; +import org.apache.avro.Protocol; +import org.apache.avro.Schema; +import org.apache.avro.compiler.specific.SpecificCompiler; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecordBuilder; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.EncoderFactory; +import org.junit.Test; + +/** + * + * @author zoly + */ +public class TestCycle { + + @Test + public void testCycleGeneration() throws ParseException, IOException { + final ClassLoader cl = Thread.currentThread().getContextClassLoader(); + Idl idl = new Idl(cl.getResourceAsStream("input/cycle.avdl"), + "UTF-8"); + Protocol protocol = idl.CompilationUnit(); + String json = protocol.toString(); + System.out.println(json); + + SpecificCompiler compiler = new SpecificCompiler(protocol); + compiler.setStringType(GenericData.StringType.String); + File output = new File("./target"); + compiler.compileToDestination(null, output); + + Map schemas = new HashMap(); + for (Schema schema : protocol.getTypes()) { + final String name = schema.getName(); + schemas.put(name, schema); + } + + GenericRecordBuilder rb2 = new GenericRecordBuilder(schemas.get("SampleNode")); + rb2.set("count", 10); + rb2.set("subNodes", Collections.EMPTY_LIST); + GenericData.Record node = rb2.build(); + + GenericRecordBuilder mb = new GenericRecordBuilder(schemas.get("Method")); + mb.set("declaringClass", "Test"); + mb.set("methodName", "test"); + GenericData.Record method = mb.build(); + + GenericRecordBuilder spb = new GenericRecordBuilder(schemas.get("SamplePair")); + spb.set("method", method); + spb.set("node", node); + GenericData.Record sp = spb.build(); + + + GenericRecordBuilder rb = new GenericRecordBuilder(schemas.get("SampleNode")); + rb.set("count", 10); + rb.set("subNodes", Arrays.asList(sp)); + GenericData.Record record = rb.build(); + + serDeserRecord(record); + + } + + private static void serDeserRecord(GenericData.Record data) throws IOException { + ByteArrayOutputStream bab = new ByteArrayOutputStream(); + GenericDatumWriter writer = new GenericDatumWriter(data.getSchema()); + final BinaryEncoder directBinaryEncoder = EncoderFactory.get().directBinaryEncoder(bab, null); + writer.write(data, directBinaryEncoder); + directBinaryEncoder.flush(); + ByteArrayInputStream bis = new ByteArrayInputStream(bab.toByteArray(), 0, bab.size()); + GenericDatumReader reader = new GenericDatumReader(data.getSchema()); + BinaryDecoder directBinaryDecoder = DecoderFactory.get().directBinaryDecoder(bis, null); + GenericData.Record read = (GenericData.Record) reader.read(null, directBinaryDecoder); + Assert.assertEquals(data.toString(), read.toString()); + } + + +} diff --git a/lang/java/ipc/nb-configuration.xml b/lang/java/ipc/nb-configuration.xml new file mode 100644 index 00000000000..93af9a7ce41 --- /dev/null +++ b/lang/java/ipc/nb-configuration.xml @@ -0,0 +1,18 @@ + + + + + + JDK_1.6 + + From 1f6fa7b79efde2eb999de34334b686a77b23e46a Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Wed, 18 Nov 2015 17:44:50 -0500 Subject: [PATCH 02/22] [add] AVRO-1723 and AVRO-1667 --- .../org/apache/avro/io/parsing/Symbol.java | 97 ++++---- .../avro/compiler/idl/SchemaResolver.java | 229 ++++++++++++++++++ .../org/apache/avro/compiler/idl/idl.jj | 7 +- .../compiler/src/test/idl/input/cycle.avdl | 24 ++ .../compiler/src/test/idl/output/cycle.avpr | 55 +++++ .../apache/avro/compiler/idl/TestCycle.java | 105 ++++++++ lang/java/ipc/nb-configuration.xml | 18 ++ 7 files changed, 489 insertions(+), 46 deletions(-) create mode 100644 lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java create mode 100644 lang/java/compiler/src/test/idl/input/cycle.avdl create mode 100644 lang/java/compiler/src/test/idl/output/cycle.avpr create mode 100644 lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestCycle.java create mode 100644 lang/java/ipc/nb-configuration.xml diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java index 08a9d14ec3f..9d733817155 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java @@ -60,7 +60,7 @@ public enum Kind { * the symbols that forms the production for this symbol. The * sequence is in the reverse order of production. This is useful * for easy copying onto parsing stack. - * + * * Please note that this is a final. So the production for a symbol * should be known before that symbol is constructed. This requirement * cannot be met for those symbols which are recursive (e.g. a record that @@ -77,8 +77,8 @@ public enum Kind { protected Symbol(Kind kind) { this(kind, null); } - - + + protected Symbol(Kind kind, Symbol[] production) { this.production = production; this.kind = kind; @@ -120,7 +120,7 @@ static Symbol alt(Symbol[] symbols, String[] labels) { static Symbol error(String e) { return new ErrorAction(e); } - + /** * A convenience method to construct a ResolvingAction. * @param w The writer symbol @@ -129,32 +129,32 @@ static Symbol error(String e) { static Symbol resolve(Symbol w, Symbol r) { return new ResolvingAction(w, r); } - + private static class Fixup { - public final Symbol[] symbols; - public final int pos; - + public Symbol[] symbols; + public int pos; + public Fixup(Symbol[] symbols, int pos) { this.symbols = symbols; this.pos = pos; } } - + public Symbol flatten(Map map, Map> map2) { return this; } - + public int flattenedSize() { return 1; } - + /** * Flattens the given sub-array of symbols into an sub-array of symbols. Every * Sequence in the input are replaced by its production recursively. * Non-Sequence symbols, they internally have other symbols * those internal symbols also get flattened. - * + * * The algorithm does a few tricks to handle recursive symbol definitions. * In order to avoid infinite recursion with recursive symbols, we have a map * of Symbol->Symbol. Before fully constructing a flattened symbol for a @@ -168,7 +168,7 @@ public int flattenedSize() { * has not not be fully constructed yet, we copy a bunch of nulls. * Fix-up remembers all those null patches. The fix-ups gets finally * filled when we know the symbols to occupy those patches. - * + * * @param in The array of input symbols to flatten * @param start The position where the input sub-array starts. * @param out The output that receives the flattened list of symbols. The @@ -190,6 +190,15 @@ static void flatten(Symbol[] in, int start, List l = map2.get(s); if (l == null) { System.arraycopy(p, 0, out, j, p.length); + // Fixups need to be relocated! + for (List value : map2.values()) { + for (Fixup fixup : value) { + if (fixup.symbols == p) { + fixup.symbols = out; + fixup.pos += j; + } + } + } } else { l.add(new Fixup(out, j)); } @@ -232,7 +241,7 @@ public Terminal(String printName) { public static class ImplicitAction extends Symbol { /** - * Set to true if and only if this implicit action is + * Set to true if and only if this implicit action is * a trailing action. That is, it is an action that follows * real symbol. E.g {@link Symbol#DEFAULT_END_ACTION}. */ @@ -241,13 +250,13 @@ public static class ImplicitAction extends Symbol { private ImplicitAction() { this(false); } - + private ImplicitAction(boolean isTrailing) { super(Kind.IMPLICIT_ACTION); this.isTrailing = isTrailing; } } - + protected static class Root extends Symbol { private Root(Symbol... symbols) { super(Kind.ROOT, makeProduction(symbols)); @@ -262,7 +271,7 @@ private static Symbol[] makeProduction(Symbol[] symbols) { return result; } } - + protected static class Sequence extends Symbol implements Iterable { private Sequence(Symbol[] productions) { super(Kind.SEQUENCE, productions); @@ -271,19 +280,19 @@ private Sequence(Symbol[] productions) { public Symbol get(int index) { return production[index]; } - + public int size() { return production.length; } - + public Iterator iterator() { return new Iterator() { private int pos = production.length; - + public boolean hasNext() { return 0 < pos; } - + public Symbol next() { if (0 < pos) { return production[--pos]; @@ -291,7 +300,7 @@ public Symbol next() { throw new NoSuchElementException(); } } - + public void remove() { throw new UnsupportedOperationException(); } @@ -306,7 +315,7 @@ public Sequence flatten(Map map, map.put(this, result); List l = new ArrayList(); map2.put(result, l); - + flatten(production, 0, result.production, 0, map, map2); for (Fixup f : l) { @@ -326,19 +335,19 @@ public final int flattenedSize() { public static class Repeater extends Symbol { public final Symbol end; - + private Repeater(Symbol end, Symbol... sequenceToRepeat) { super(Kind.REPEATER, makeProduction(sequenceToRepeat)); this.end = end; production[0] = this; } - + private static Symbol[] makeProduction(Symbol[] p) { Symbol[] result = new Symbol[p.length + 1]; System.arraycopy(p, 0, result, 1, p.length); return result; } - + @Override public Repeater flatten(Map map, Map> map2) { @@ -349,9 +358,9 @@ public Repeater flatten(Map map, } } - + /** - * Returns true if the Parser contains any Error symbol, indicating that it may fail + * Returns true if the Parser contains any Error symbol, indicating that it may fail * for some inputs. */ public static boolean hasErrors(Symbol symbol) { @@ -374,7 +383,7 @@ public static boolean hasErrors(Symbol symbol) { throw new RuntimeException("unknown symbol kind: " + symbol.kind); } } - + private static boolean hasErrors(Symbol root, Symbol[] symbols) { if(null != symbols) { for(Symbol s: symbols) { @@ -388,7 +397,7 @@ private static boolean hasErrors(Symbol root, Symbol[] symbols) { } return false; } - + public static class Alternative extends Symbol { public final Symbol[] symbols; public final String[] labels; @@ -397,15 +406,15 @@ private Alternative(Symbol[] symbols, String[] labels) { this.symbols = symbols; this.labels = labels; } - + public Symbol getSymbol(int index) { return symbols[index]; } - + public String getLabel(int index) { return labels[index]; } - + public int size() { return symbols.length; } @@ -454,7 +463,7 @@ public static class IntCheckAction extends Symbol { public static EnumAdjustAction enumAdjustAction(int rsymCount, Object[] adj) { return new EnumAdjustAction(rsymCount, adj); } - + public static class EnumAdjustAction extends IntCheckAction { public final Object[] adjustments; @Deprecated public EnumAdjustAction(int rsymCount, Object[] adjustments) { @@ -478,7 +487,7 @@ private ResolvingAction(Symbol writer, Symbol reader) { this.writer = writer; this.reader = reader; } - + @Override public ResolvingAction flatten(Map map, Map> map2) { @@ -487,7 +496,7 @@ public ResolvingAction flatten(Map map, } } - + public static SkipAction skipAction(Symbol symToSkip) { return new SkipAction(symToSkip); } @@ -498,7 +507,7 @@ public static class SkipAction extends ImplicitAction { super(true); this.symToSkip = symToSkip; } - + @Override public SkipAction flatten(Map map, Map> map2) { @@ -510,7 +519,7 @@ public SkipAction flatten(Map map, public static FieldAdjustAction fieldAdjustAction(int rindex, String fname) { return new FieldAdjustAction(rindex, fname); } - + public static class FieldAdjustAction extends ImplicitAction { public final int rindex; public final String fname; @@ -519,7 +528,7 @@ public static class FieldAdjustAction extends ImplicitAction { this.fname = fname; } } - + public static FieldOrderAction fieldOrderAction(Schema.Field[] fields) { return new FieldOrderAction(fields); } @@ -553,13 +562,13 @@ public static class UnionAdjustAction extends ImplicitAction { this.rindex = rindex; this.symToParse = symToParse; } - + @Override public UnionAdjustAction flatten(Map map, Map> map2) { return new UnionAdjustAction(rindex, symToParse.flatten(map, map2)); } - + } /** For JSON. */ @@ -573,11 +582,11 @@ public static class EnumLabelsAction extends IntCheckAction { super(symbols.size()); this.symbols = symbols; } - + public String getLabel(int n) { return symbols.get(n); } - + public int findLabel(String l) { if (l != null) { for (int i = 0; i < symbols.size(); i++) { @@ -619,7 +628,7 @@ public int findLabel(String l) { public static final Symbol RECORD_END = new ImplicitAction(true); public static final Symbol UNION_END = new ImplicitAction(true); public static final Symbol FIELD_END = new ImplicitAction(true); - + public static final Symbol DEFAULT_END_ACTION = new ImplicitAction(true); public static final Symbol MAP_KEY_MARKER = new Symbol.Terminal("map-key-marker"); diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java new file mode 100644 index 00000000000..df0c7ac2ef5 --- /dev/null +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java @@ -0,0 +1,229 @@ +/* + * Copyright 2015 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.compiler.idl; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.avro.JsonProperties; +import org.apache.avro.LogicalType; +import org.apache.avro.Protocol; +import org.apache.avro.Schema; +import org.codehaus.jackson.JsonNode; + +/** + * + * @author zoly + */ +final class SchemaResolver { + + private SchemaResolver() { } + + private static final String UR_SCHEMA_ATTR = "org.apache.avro.compiler.idl.unresolved.name"; + + static Schema unresolvedSchema(final String name) { + Schema schema = Schema.createRecord("UnresolvedSchema", "unresolved schema", + "org.apache.avro.compiler", false, Collections.EMPTY_LIST); + schema.addProp(UR_SCHEMA_ATTR, name); + return schema; + } + + static boolean isUnresolvedSchema(final Schema schema) { + return (schema.getType() == Schema.Type.RECORD && schema.getProp(UR_SCHEMA_ATTR) != null); + } + + static String getUnresolvedSchemaName(final Schema schema) { + String name = schema.getProp(UR_SCHEMA_ATTR); + if (name == null) { + throw new IllegalArgumentException("Schema " + schema + " is not a unresolved schema"); + } else { + return name; + } + } + + + + static Protocol resolve(final Protocol protocol) { + Protocol result = new Protocol(protocol.getName(), protocol.getDoc(), protocol.getNamespace()); + final Collection types = protocol.getTypes(); + List newSchemas = new ArrayList(types.size()); + Map processed = new HashMap(); + for (Schema schema : types) { + newSchemas.add(resolve(schema, protocol, processed)); + } + result.setTypes(newSchemas); + + for (Map.Entry entry : protocol.getMessages().entrySet()) { + Protocol.Message value = entry.getValue(); + Protocol.Message nvalue; + if (value.isOneWay()) { + Schema request = value.getRequest(); + nvalue = result.createMessage(value.getName(), value.getDoc(), + value.getObjectProps(), intern(request, processed)); + } else { + Schema request = value.getRequest(); + Schema response = value.getResponse(); + Schema errors = value.getErrors(); + nvalue = result.createMessage(value.getName(), value.getDoc(), + value.getObjectProps(), intern(request, processed), + intern(response, processed), intern(errors, processed)); + } + result.getMessages().put(entry.getKey(), nvalue); + } + copyProps(protocol, result); + return result; + } + + private static void copyProps(final JsonProperties from, final JsonProperties to) { + for (Map.Entry entry : from.getJsonProps().entrySet()) { + to.addProp(entry.getKey(), entry.getValue()); + } + } + + + static Schema resolve(final Schema schema, final Protocol protocol, final Map processed) { + final String fullName = schema.getFullName(); + if (fullName != null && processed.containsKey(fullName)) { + return processed.get(schema.getFullName()); + } else if (isUnresolvedSchema(schema)) { + final String unresolvedSchemaName = getUnresolvedSchemaName(schema); + Schema type = protocol.getType(unresolvedSchemaName); + if (type == null) { + throw new IllegalArgumentException("Cannot resolve " + unresolvedSchemaName); + } + return resolve(type, protocol, processed); + } else { + switch (schema.getType()) { + case RECORD: + Schema createRecord = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), + schema.isError()); + processed.put(schema.getFullName(), createRecord); + final List currFields = schema.getFields(); + List newFields = new ArrayList(currFields.size()); + for (Schema.Field field : currFields) { + Schema.Field nf = new Schema.Field(field.name(), resolve(field.schema(), protocol, processed), + field.doc(), field.defaultVal(), field.order()); + for (String alias : field.aliases()) { + nf.addAlias(alias); + } + newFields.add(nf); + } + createRecord.setFields(newFields); + final LogicalType lt = schema.getLogicalType(); + if (lt != null) { + lt.addToSchema(createRecord); + } + copyProps(schema, createRecord); + return createRecord; + case MAP: + Schema result = Schema.createMap(resolve(schema.getValueType(), protocol, processed)); + copyProps(schema, result); + return result; + case ARRAY: + Schema aresult = Schema.createArray(resolve(schema.getElementType(), protocol, processed)); + copyProps(schema, aresult); + return aresult; + case UNION: + final List uTypes = schema.getTypes(); + List newTypes = new ArrayList(uTypes.size()); + for (Schema s : uTypes) { + newTypes.add(resolve(s, protocol, processed)); + } + Schema bresult = Schema.createUnion(newTypes); + copyProps(schema, bresult); + return bresult; + case ENUM: + case FIXED: + case STRING: + case BYTES: + case INT: + case LONG: + case FLOAT: + case DOUBLE: + case BOOLEAN: + case NULL: + return schema; + default: + throw new RuntimeException("Unknown type: " + schema); + } + } + } + + public static Schema intern(final Schema schema, final Map processed) { + if (schema == null) { + return null; + } + final String fullName = schema.getFullName(); + if (fullName != null && processed.containsKey(fullName)) { + return processed.get(schema.getFullName()); + } else { + switch (schema.getType()) { + case RECORD: + Schema createRecord = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), + schema.isError()); + processed.put(schema.getFullName(), createRecord); + final List currFields = schema.getFields(); + List newFields = new ArrayList(currFields.size()); + for (Schema.Field field : currFields) { + Schema.Field nf = new Schema.Field(field.name(), intern(field.schema(), processed), + field.doc(), field.defaultVal(), field.order()); + for (String alias : field.aliases()) { + nf.addAlias(alias); + } + newFields.add(nf); + } + createRecord.setFields(newFields); + final LogicalType lt = schema.getLogicalType(); + if (lt != null) { + lt.addToSchema(createRecord); + } + copyProps(schema, createRecord); + return createRecord; + case MAP: + return Schema.createMap(intern(schema.getValueType(), processed)); + case ARRAY: + return Schema.createArray(intern(schema.getElementType(), processed)); + case UNION: + final List uTypes = schema.getTypes(); + List newTypes = new ArrayList(uTypes.size()); + for (Schema s : uTypes) { + newTypes.add(intern(s, processed)); + } + return Schema.createUnion(newTypes); + case ENUM: + case FIXED: + case STRING: + case BYTES: + case INT: + case LONG: + case FLOAT: + case DOUBLE: + case BOOLEAN: + case NULL: + return schema; + default: + throw new RuntimeException("Unknown type: " + schema); + } + } + } + + + + +} diff --git a/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj b/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj index 8f60b83b90b..a8f06234625 100644 --- a/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj +++ b/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj @@ -999,7 +999,7 @@ Protocol CompilationUnit(): ( < "\u001a" > )? ( )? - { return p; } + { return SchemaResolver.resolve(p); } } /* @@ -1465,7 +1465,10 @@ Schema ReferenceType(): name = namespace + "." + name; Schema type = names.get(name); if (type == null) - throw error("Undefined name '" + name + "'", token); + //throw error("Undefined name '" + name + "'", token); + { + type = SchemaResolver.unresolvedSchema(name); + } return type; } } diff --git a/lang/java/compiler/src/test/idl/input/cycle.avdl b/lang/java/compiler/src/test/idl/input/cycle.avdl new file mode 100644 index 00000000000..f4344316024 --- /dev/null +++ b/lang/java/compiler/src/test/idl/input/cycle.avdl @@ -0,0 +1,24 @@ +@namespace("org.apache.avro.gen") +protocol Cycle { + + record SampleNode { + int count = 0; + array subNodes; + } + + record Method { + string declaringClass; + string methodName; + } + + record SamplePair { + Method method; + SampleNode node; + } + + record SelfRef { + string something; + array subNodes = []; + } + +} diff --git a/lang/java/compiler/src/test/idl/output/cycle.avpr b/lang/java/compiler/src/test/idl/output/cycle.avpr new file mode 100644 index 00000000000..53658afd00e --- /dev/null +++ b/lang/java/compiler/src/test/idl/output/cycle.avpr @@ -0,0 +1,55 @@ +{ + "protocol" : "Cycle", + "namespace" : "org.apache.avro.gen", + "types" : [ { + "type" : "record", + "name" : "SampleNode", + "fields" : [ { + "name" : "count", + "type" : "int", + "default" : 0 + }, { + "name" : "subNodes", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "SamplePair", + "fields" : [ { + "name" : "method", + "type" : { + "type" : "record", + "name" : "Method", + "fields" : [ { + "name" : "declaringClass", + "type" : "string" + }, { + "name" : "methodName", + "type" : "string" + } ] + } + }, { + "name" : "node", + "type" : "SampleNode" + } ] + } + } + } ] + }, { + "type" : "record", + "name" : "SelfRef", + "fields" : [ { + "name" : "something", + "type" : "string" + }, { + "name" : "subNodes", + "type" : { + "type" : "array", + "items" : "SelfRef" + }, + "default" : [ ] + } ] + } ], + "messages" : { } +} + diff --git a/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestCycle.java b/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestCycle.java new file mode 100644 index 00000000000..a38e978ef7d --- /dev/null +++ b/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestCycle.java @@ -0,0 +1,105 @@ +/* + * Copyright 2015 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.compiler.idl; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import junit.framework.Assert; +import org.apache.avro.Protocol; +import org.apache.avro.Schema; +import org.apache.avro.compiler.specific.SpecificCompiler; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecordBuilder; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.EncoderFactory; +import org.junit.Test; + +/** + * + * @author zoly + */ +public class TestCycle { + + @Test + public void testCycleGeneration() throws ParseException, IOException { + final ClassLoader cl = Thread.currentThread().getContextClassLoader(); + Idl idl = new Idl(cl.getResourceAsStream("input/cycle.avdl"), + "UTF-8"); + Protocol protocol = idl.CompilationUnit(); + String json = protocol.toString(); + System.out.println(json); + + SpecificCompiler compiler = new SpecificCompiler(protocol); + compiler.setStringType(GenericData.StringType.String); + File output = new File("./target"); + compiler.compileToDestination(null, output); + + Map schemas = new HashMap(); + for (Schema schema : protocol.getTypes()) { + final String name = schema.getName(); + schemas.put(name, schema); + } + + GenericRecordBuilder rb2 = new GenericRecordBuilder(schemas.get("SampleNode")); + rb2.set("count", 10); + rb2.set("subNodes", Collections.EMPTY_LIST); + GenericData.Record node = rb2.build(); + + GenericRecordBuilder mb = new GenericRecordBuilder(schemas.get("Method")); + mb.set("declaringClass", "Test"); + mb.set("methodName", "test"); + GenericData.Record method = mb.build(); + + GenericRecordBuilder spb = new GenericRecordBuilder(schemas.get("SamplePair")); + spb.set("method", method); + spb.set("node", node); + GenericData.Record sp = spb.build(); + + + GenericRecordBuilder rb = new GenericRecordBuilder(schemas.get("SampleNode")); + rb.set("count", 10); + rb.set("subNodes", Arrays.asList(sp)); + GenericData.Record record = rb.build(); + + serDeserRecord(record); + + } + + private static void serDeserRecord(GenericData.Record data) throws IOException { + ByteArrayOutputStream bab = new ByteArrayOutputStream(); + GenericDatumWriter writer = new GenericDatumWriter(data.getSchema()); + final BinaryEncoder directBinaryEncoder = EncoderFactory.get().directBinaryEncoder(bab, null); + writer.write(data, directBinaryEncoder); + directBinaryEncoder.flush(); + ByteArrayInputStream bis = new ByteArrayInputStream(bab.toByteArray(), 0, bab.size()); + GenericDatumReader reader = new GenericDatumReader(data.getSchema()); + BinaryDecoder directBinaryDecoder = DecoderFactory.get().directBinaryDecoder(bis, null); + GenericData.Record read = (GenericData.Record) reader.read(null, directBinaryDecoder); + Assert.assertEquals(data.toString(), read.toString()); + } + + +} diff --git a/lang/java/ipc/nb-configuration.xml b/lang/java/ipc/nb-configuration.xml new file mode 100644 index 00000000000..93af9a7ce41 --- /dev/null +++ b/lang/java/ipc/nb-configuration.xml @@ -0,0 +1,18 @@ + + + + + + JDK_1.6 + + From fd6c4228fbecc6eff3e3ecd5ee27d2ceb46339ff Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Wed, 18 Nov 2015 17:44:50 -0500 Subject: [PATCH 03/22] [add] AVRO-1723 and AVRO-1667 --- .../org/apache/avro/io/parsing/Symbol.java | 88 +++---- .../avro/compiler/idl/SchemaResolver.java | 229 ++++++++++++++++++ .../org/apache/avro/compiler/idl/idl.jj | 7 +- .../compiler/src/test/idl/input/cycle.avdl | 24 ++ .../compiler/src/test/idl/output/cycle.avpr | 55 +++++ .../apache/avro/compiler/idl/TestCycle.java | 105 ++++++++ lang/java/ipc/nb-configuration.xml | 18 ++ 7 files changed, 480 insertions(+), 46 deletions(-) create mode 100644 lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java create mode 100644 lang/java/compiler/src/test/idl/input/cycle.avdl create mode 100644 lang/java/compiler/src/test/idl/output/cycle.avpr create mode 100644 lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestCycle.java create mode 100644 lang/java/ipc/nb-configuration.xml diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java index 80ae6442b47..11e07376db2 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java @@ -60,7 +60,7 @@ public enum Kind { * the symbols that forms the production for this symbol. The * sequence is in the reverse order of production. This is useful * for easy copying onto parsing stack. - * + * * Please note that this is a final. So the production for a symbol * should be known before that symbol is constructed. This requirement * cannot be met for those symbols which are recursive (e.g. a record that @@ -77,8 +77,8 @@ public enum Kind { protected Symbol(Kind kind) { this(kind, null); } - - + + protected Symbol(Kind kind, Symbol[] production) { this.production = production; this.kind = kind; @@ -120,7 +120,7 @@ static Symbol alt(Symbol[] symbols, String[] labels) { static Symbol error(String e) { return new ErrorAction(e); } - + /** * A convenience method to construct a ResolvingAction. * @param w The writer symbol @@ -129,32 +129,32 @@ static Symbol error(String e) { static Symbol resolve(Symbol w, Symbol r) { return new ResolvingAction(w, r); } - + private static class Fixup { - public final Symbol[] symbols; - public final int pos; - + public Symbol[] symbols; + public int pos; + public Fixup(Symbol[] symbols, int pos) { this.symbols = symbols; this.pos = pos; } } - + public Symbol flatten(Map map, Map> map2) { return this; } - + public int flattenedSize() { return 1; } - + /** * Flattens the given sub-array of symbols into an sub-array of symbols. Every * Sequence in the input are replaced by its production recursively. * Non-Sequence symbols, they internally have other symbols * those internal symbols also get flattened. - * + * * The algorithm does a few tricks to handle recursive symbol definitions. * In order to avoid infinite recursion with recursive symbols, we have a map * of Symbol->Symbol. Before fully constructing a flattened symbol for a @@ -168,7 +168,7 @@ public int flattenedSize() { * has not not be fully constructed yet, we copy a bunch of nulls. * Fix-up remembers all those null patches. The fix-ups gets finally * filled when we know the symbols to occupy those patches. - * + * * @param in The array of input symbols to flatten * @param start The position where the input sub-array starts. * @param out The output that receives the flattened list of symbols. The @@ -246,7 +246,7 @@ public Terminal(String printName) { public static class ImplicitAction extends Symbol { /** - * Set to true if and only if this implicit action is + * Set to true if and only if this implicit action is * a trailing action. That is, it is an action that follows * real symbol. E.g {@link Symbol#DEFAULT_END_ACTION}. */ @@ -255,13 +255,13 @@ public static class ImplicitAction extends Symbol { private ImplicitAction() { this(false); } - + private ImplicitAction(boolean isTrailing) { super(Kind.IMPLICIT_ACTION); this.isTrailing = isTrailing; } } - + protected static class Root extends Symbol { private Root(Symbol... symbols) { super(Kind.ROOT, makeProduction(symbols)); @@ -276,7 +276,7 @@ private static Symbol[] makeProduction(Symbol[] symbols) { return result; } } - + protected static class Sequence extends Symbol implements Iterable { private Sequence(Symbol[] productions) { super(Kind.SEQUENCE, productions); @@ -285,19 +285,19 @@ private Sequence(Symbol[] productions) { public Symbol get(int index) { return production[index]; } - + public int size() { return production.length; } - + public Iterator iterator() { return new Iterator() { private int pos = production.length; - + public boolean hasNext() { return 0 < pos; } - + public Symbol next() { if (0 < pos) { return production[--pos]; @@ -305,7 +305,7 @@ public Symbol next() { throw new NoSuchElementException(); } } - + public void remove() { throw new UnsupportedOperationException(); } @@ -320,7 +320,7 @@ public Sequence flatten(Map map, map.put(this, result); List l = new ArrayList(); map2.put(result, l); - + flatten(production, 0, result.production, 0, map, map2); for (Fixup f : l) { @@ -340,19 +340,19 @@ public final int flattenedSize() { public static class Repeater extends Symbol { public final Symbol end; - + private Repeater(Symbol end, Symbol... sequenceToRepeat) { super(Kind.REPEATER, makeProduction(sequenceToRepeat)); this.end = end; production[0] = this; } - + private static Symbol[] makeProduction(Symbol[] p) { Symbol[] result = new Symbol[p.length + 1]; System.arraycopy(p, 0, result, 1, p.length); return result; } - + @Override public Repeater flatten(Map map, Map> map2) { @@ -363,9 +363,9 @@ public Repeater flatten(Map map, } } - + /** - * Returns true if the Parser contains any Error symbol, indicating that it may fail + * Returns true if the Parser contains any Error symbol, indicating that it may fail * for some inputs. */ public static boolean hasErrors(Symbol symbol) { @@ -388,7 +388,7 @@ public static boolean hasErrors(Symbol symbol) { throw new RuntimeException("unknown symbol kind: " + symbol.kind); } } - + private static boolean hasErrors(Symbol root, Symbol[] symbols) { if(null != symbols) { for(Symbol s: symbols) { @@ -402,7 +402,7 @@ private static boolean hasErrors(Symbol root, Symbol[] symbols) { } return false; } - + public static class Alternative extends Symbol { public final Symbol[] symbols; public final String[] labels; @@ -411,15 +411,15 @@ private Alternative(Symbol[] symbols, String[] labels) { this.symbols = symbols; this.labels = labels; } - + public Symbol getSymbol(int index) { return symbols[index]; } - + public String getLabel(int index) { return labels[index]; } - + public int size() { return symbols.length; } @@ -468,7 +468,7 @@ public static class IntCheckAction extends Symbol { public static EnumAdjustAction enumAdjustAction(int rsymCount, Object[] adj) { return new EnumAdjustAction(rsymCount, adj); } - + public static class EnumAdjustAction extends IntCheckAction { public final Object[] adjustments; @Deprecated public EnumAdjustAction(int rsymCount, Object[] adjustments) { @@ -492,7 +492,7 @@ private ResolvingAction(Symbol writer, Symbol reader) { this.writer = writer; this.reader = reader; } - + @Override public ResolvingAction flatten(Map map, Map> map2) { @@ -501,7 +501,7 @@ public ResolvingAction flatten(Map map, } } - + public static SkipAction skipAction(Symbol symToSkip) { return new SkipAction(symToSkip); } @@ -512,7 +512,7 @@ public static class SkipAction extends ImplicitAction { super(true); this.symToSkip = symToSkip; } - + @Override public SkipAction flatten(Map map, Map> map2) { @@ -524,7 +524,7 @@ public SkipAction flatten(Map map, public static FieldAdjustAction fieldAdjustAction(int rindex, String fname) { return new FieldAdjustAction(rindex, fname); } - + public static class FieldAdjustAction extends ImplicitAction { public final int rindex; public final String fname; @@ -533,7 +533,7 @@ public static class FieldAdjustAction extends ImplicitAction { this.fname = fname; } } - + public static FieldOrderAction fieldOrderAction(Schema.Field[] fields) { return new FieldOrderAction(fields); } @@ -567,13 +567,13 @@ public static class UnionAdjustAction extends ImplicitAction { this.rindex = rindex; this.symToParse = symToParse; } - + @Override public UnionAdjustAction flatten(Map map, Map> map2) { return new UnionAdjustAction(rindex, symToParse.flatten(map, map2)); } - + } /** For JSON. */ @@ -587,11 +587,11 @@ public static class EnumLabelsAction extends IntCheckAction { super(symbols.size()); this.symbols = symbols; } - + public String getLabel(int n) { return symbols.get(n); } - + public int findLabel(String l) { if (l != null) { for (int i = 0; i < symbols.size(); i++) { @@ -633,7 +633,7 @@ public int findLabel(String l) { public static final Symbol RECORD_END = new ImplicitAction(true); public static final Symbol UNION_END = new ImplicitAction(true); public static final Symbol FIELD_END = new ImplicitAction(true); - + public static final Symbol DEFAULT_END_ACTION = new ImplicitAction(true); public static final Symbol MAP_KEY_MARKER = new Symbol.Terminal("map-key-marker"); diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java new file mode 100644 index 00000000000..df0c7ac2ef5 --- /dev/null +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java @@ -0,0 +1,229 @@ +/* + * Copyright 2015 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.compiler.idl; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.avro.JsonProperties; +import org.apache.avro.LogicalType; +import org.apache.avro.Protocol; +import org.apache.avro.Schema; +import org.codehaus.jackson.JsonNode; + +/** + * + * @author zoly + */ +final class SchemaResolver { + + private SchemaResolver() { } + + private static final String UR_SCHEMA_ATTR = "org.apache.avro.compiler.idl.unresolved.name"; + + static Schema unresolvedSchema(final String name) { + Schema schema = Schema.createRecord("UnresolvedSchema", "unresolved schema", + "org.apache.avro.compiler", false, Collections.EMPTY_LIST); + schema.addProp(UR_SCHEMA_ATTR, name); + return schema; + } + + static boolean isUnresolvedSchema(final Schema schema) { + return (schema.getType() == Schema.Type.RECORD && schema.getProp(UR_SCHEMA_ATTR) != null); + } + + static String getUnresolvedSchemaName(final Schema schema) { + String name = schema.getProp(UR_SCHEMA_ATTR); + if (name == null) { + throw new IllegalArgumentException("Schema " + schema + " is not a unresolved schema"); + } else { + return name; + } + } + + + + static Protocol resolve(final Protocol protocol) { + Protocol result = new Protocol(protocol.getName(), protocol.getDoc(), protocol.getNamespace()); + final Collection types = protocol.getTypes(); + List newSchemas = new ArrayList(types.size()); + Map processed = new HashMap(); + for (Schema schema : types) { + newSchemas.add(resolve(schema, protocol, processed)); + } + result.setTypes(newSchemas); + + for (Map.Entry entry : protocol.getMessages().entrySet()) { + Protocol.Message value = entry.getValue(); + Protocol.Message nvalue; + if (value.isOneWay()) { + Schema request = value.getRequest(); + nvalue = result.createMessage(value.getName(), value.getDoc(), + value.getObjectProps(), intern(request, processed)); + } else { + Schema request = value.getRequest(); + Schema response = value.getResponse(); + Schema errors = value.getErrors(); + nvalue = result.createMessage(value.getName(), value.getDoc(), + value.getObjectProps(), intern(request, processed), + intern(response, processed), intern(errors, processed)); + } + result.getMessages().put(entry.getKey(), nvalue); + } + copyProps(protocol, result); + return result; + } + + private static void copyProps(final JsonProperties from, final JsonProperties to) { + for (Map.Entry entry : from.getJsonProps().entrySet()) { + to.addProp(entry.getKey(), entry.getValue()); + } + } + + + static Schema resolve(final Schema schema, final Protocol protocol, final Map processed) { + final String fullName = schema.getFullName(); + if (fullName != null && processed.containsKey(fullName)) { + return processed.get(schema.getFullName()); + } else if (isUnresolvedSchema(schema)) { + final String unresolvedSchemaName = getUnresolvedSchemaName(schema); + Schema type = protocol.getType(unresolvedSchemaName); + if (type == null) { + throw new IllegalArgumentException("Cannot resolve " + unresolvedSchemaName); + } + return resolve(type, protocol, processed); + } else { + switch (schema.getType()) { + case RECORD: + Schema createRecord = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), + schema.isError()); + processed.put(schema.getFullName(), createRecord); + final List currFields = schema.getFields(); + List newFields = new ArrayList(currFields.size()); + for (Schema.Field field : currFields) { + Schema.Field nf = new Schema.Field(field.name(), resolve(field.schema(), protocol, processed), + field.doc(), field.defaultVal(), field.order()); + for (String alias : field.aliases()) { + nf.addAlias(alias); + } + newFields.add(nf); + } + createRecord.setFields(newFields); + final LogicalType lt = schema.getLogicalType(); + if (lt != null) { + lt.addToSchema(createRecord); + } + copyProps(schema, createRecord); + return createRecord; + case MAP: + Schema result = Schema.createMap(resolve(schema.getValueType(), protocol, processed)); + copyProps(schema, result); + return result; + case ARRAY: + Schema aresult = Schema.createArray(resolve(schema.getElementType(), protocol, processed)); + copyProps(schema, aresult); + return aresult; + case UNION: + final List uTypes = schema.getTypes(); + List newTypes = new ArrayList(uTypes.size()); + for (Schema s : uTypes) { + newTypes.add(resolve(s, protocol, processed)); + } + Schema bresult = Schema.createUnion(newTypes); + copyProps(schema, bresult); + return bresult; + case ENUM: + case FIXED: + case STRING: + case BYTES: + case INT: + case LONG: + case FLOAT: + case DOUBLE: + case BOOLEAN: + case NULL: + return schema; + default: + throw new RuntimeException("Unknown type: " + schema); + } + } + } + + public static Schema intern(final Schema schema, final Map processed) { + if (schema == null) { + return null; + } + final String fullName = schema.getFullName(); + if (fullName != null && processed.containsKey(fullName)) { + return processed.get(schema.getFullName()); + } else { + switch (schema.getType()) { + case RECORD: + Schema createRecord = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), + schema.isError()); + processed.put(schema.getFullName(), createRecord); + final List currFields = schema.getFields(); + List newFields = new ArrayList(currFields.size()); + for (Schema.Field field : currFields) { + Schema.Field nf = new Schema.Field(field.name(), intern(field.schema(), processed), + field.doc(), field.defaultVal(), field.order()); + for (String alias : field.aliases()) { + nf.addAlias(alias); + } + newFields.add(nf); + } + createRecord.setFields(newFields); + final LogicalType lt = schema.getLogicalType(); + if (lt != null) { + lt.addToSchema(createRecord); + } + copyProps(schema, createRecord); + return createRecord; + case MAP: + return Schema.createMap(intern(schema.getValueType(), processed)); + case ARRAY: + return Schema.createArray(intern(schema.getElementType(), processed)); + case UNION: + final List uTypes = schema.getTypes(); + List newTypes = new ArrayList(uTypes.size()); + for (Schema s : uTypes) { + newTypes.add(intern(s, processed)); + } + return Schema.createUnion(newTypes); + case ENUM: + case FIXED: + case STRING: + case BYTES: + case INT: + case LONG: + case FLOAT: + case DOUBLE: + case BOOLEAN: + case NULL: + return schema; + default: + throw new RuntimeException("Unknown type: " + schema); + } + } + } + + + + +} diff --git a/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj b/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj index 8f60b83b90b..a8f06234625 100644 --- a/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj +++ b/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj @@ -999,7 +999,7 @@ Protocol CompilationUnit(): ( < "\u001a" > )? ( )? - { return p; } + { return SchemaResolver.resolve(p); } } /* @@ -1465,7 +1465,10 @@ Schema ReferenceType(): name = namespace + "." + name; Schema type = names.get(name); if (type == null) - throw error("Undefined name '" + name + "'", token); + //throw error("Undefined name '" + name + "'", token); + { + type = SchemaResolver.unresolvedSchema(name); + } return type; } } diff --git a/lang/java/compiler/src/test/idl/input/cycle.avdl b/lang/java/compiler/src/test/idl/input/cycle.avdl new file mode 100644 index 00000000000..f4344316024 --- /dev/null +++ b/lang/java/compiler/src/test/idl/input/cycle.avdl @@ -0,0 +1,24 @@ +@namespace("org.apache.avro.gen") +protocol Cycle { + + record SampleNode { + int count = 0; + array subNodes; + } + + record Method { + string declaringClass; + string methodName; + } + + record SamplePair { + Method method; + SampleNode node; + } + + record SelfRef { + string something; + array subNodes = []; + } + +} diff --git a/lang/java/compiler/src/test/idl/output/cycle.avpr b/lang/java/compiler/src/test/idl/output/cycle.avpr new file mode 100644 index 00000000000..53658afd00e --- /dev/null +++ b/lang/java/compiler/src/test/idl/output/cycle.avpr @@ -0,0 +1,55 @@ +{ + "protocol" : "Cycle", + "namespace" : "org.apache.avro.gen", + "types" : [ { + "type" : "record", + "name" : "SampleNode", + "fields" : [ { + "name" : "count", + "type" : "int", + "default" : 0 + }, { + "name" : "subNodes", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "SamplePair", + "fields" : [ { + "name" : "method", + "type" : { + "type" : "record", + "name" : "Method", + "fields" : [ { + "name" : "declaringClass", + "type" : "string" + }, { + "name" : "methodName", + "type" : "string" + } ] + } + }, { + "name" : "node", + "type" : "SampleNode" + } ] + } + } + } ] + }, { + "type" : "record", + "name" : "SelfRef", + "fields" : [ { + "name" : "something", + "type" : "string" + }, { + "name" : "subNodes", + "type" : { + "type" : "array", + "items" : "SelfRef" + }, + "default" : [ ] + } ] + } ], + "messages" : { } +} + diff --git a/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestCycle.java b/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestCycle.java new file mode 100644 index 00000000000..a38e978ef7d --- /dev/null +++ b/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestCycle.java @@ -0,0 +1,105 @@ +/* + * Copyright 2015 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.compiler.idl; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import junit.framework.Assert; +import org.apache.avro.Protocol; +import org.apache.avro.Schema; +import org.apache.avro.compiler.specific.SpecificCompiler; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecordBuilder; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.EncoderFactory; +import org.junit.Test; + +/** + * + * @author zoly + */ +public class TestCycle { + + @Test + public void testCycleGeneration() throws ParseException, IOException { + final ClassLoader cl = Thread.currentThread().getContextClassLoader(); + Idl idl = new Idl(cl.getResourceAsStream("input/cycle.avdl"), + "UTF-8"); + Protocol protocol = idl.CompilationUnit(); + String json = protocol.toString(); + System.out.println(json); + + SpecificCompiler compiler = new SpecificCompiler(protocol); + compiler.setStringType(GenericData.StringType.String); + File output = new File("./target"); + compiler.compileToDestination(null, output); + + Map schemas = new HashMap(); + for (Schema schema : protocol.getTypes()) { + final String name = schema.getName(); + schemas.put(name, schema); + } + + GenericRecordBuilder rb2 = new GenericRecordBuilder(schemas.get("SampleNode")); + rb2.set("count", 10); + rb2.set("subNodes", Collections.EMPTY_LIST); + GenericData.Record node = rb2.build(); + + GenericRecordBuilder mb = new GenericRecordBuilder(schemas.get("Method")); + mb.set("declaringClass", "Test"); + mb.set("methodName", "test"); + GenericData.Record method = mb.build(); + + GenericRecordBuilder spb = new GenericRecordBuilder(schemas.get("SamplePair")); + spb.set("method", method); + spb.set("node", node); + GenericData.Record sp = spb.build(); + + + GenericRecordBuilder rb = new GenericRecordBuilder(schemas.get("SampleNode")); + rb.set("count", 10); + rb.set("subNodes", Arrays.asList(sp)); + GenericData.Record record = rb.build(); + + serDeserRecord(record); + + } + + private static void serDeserRecord(GenericData.Record data) throws IOException { + ByteArrayOutputStream bab = new ByteArrayOutputStream(); + GenericDatumWriter writer = new GenericDatumWriter(data.getSchema()); + final BinaryEncoder directBinaryEncoder = EncoderFactory.get().directBinaryEncoder(bab, null); + writer.write(data, directBinaryEncoder); + directBinaryEncoder.flush(); + ByteArrayInputStream bis = new ByteArrayInputStream(bab.toByteArray(), 0, bab.size()); + GenericDatumReader reader = new GenericDatumReader(data.getSchema()); + BinaryDecoder directBinaryDecoder = DecoderFactory.get().directBinaryDecoder(bis, null); + GenericData.Record read = (GenericData.Record) reader.read(null, directBinaryDecoder); + Assert.assertEquals(data.toString(), read.toString()); + } + + +} diff --git a/lang/java/ipc/nb-configuration.xml b/lang/java/ipc/nb-configuration.xml new file mode 100644 index 00000000000..93af9a7ce41 --- /dev/null +++ b/lang/java/ipc/nb-configuration.xml @@ -0,0 +1,18 @@ + + + + + + JDK_1.6 + + From 3ff1fa8cac828bfaabd082cb82d7ea3a3827f02f Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Thu, 17 Mar 2016 10:18:02 -0400 Subject: [PATCH 04/22] Merge origin/trunk-AVRO-1723 into trunk-AVRO-1723 Conflicts: lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java --- .../src/main/java/org/apache/avro/io/parsing/Symbol.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java index 11e07376db2..6fb4ca0bc10 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java @@ -131,8 +131,8 @@ static Symbol resolve(Symbol w, Symbol r) { } private static class Fixup { - public Symbol[] symbols; - public int pos; + public final Symbol[] symbols; + public final int pos; public Fixup(Symbol[] symbols, int pos) { this.symbols = symbols; @@ -638,4 +638,3 @@ public int findLabel(String l) { public static final Symbol MAP_KEY_MARKER = new Symbol.Terminal("map-key-marker"); } - From ce16a3fc9e493d50425ecc697e397c486e9c62dd Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Thu, 17 Mar 2016 10:28:44 -0400 Subject: [PATCH 05/22] Merge origin/trunk-AVRO-1723 into trunk-AVRO-1723 Conflicts: lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java --- .../org/apache/avro/io/parsing/Symbol.java | 85 ++++++++++--------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java index 6fb4ca0bc10..80ae6442b47 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java @@ -60,7 +60,7 @@ public enum Kind { * the symbols that forms the production for this symbol. The * sequence is in the reverse order of production. This is useful * for easy copying onto parsing stack. - * + * * Please note that this is a final. So the production for a symbol * should be known before that symbol is constructed. This requirement * cannot be met for those symbols which are recursive (e.g. a record that @@ -77,8 +77,8 @@ public enum Kind { protected Symbol(Kind kind) { this(kind, null); } - - + + protected Symbol(Kind kind, Symbol[] production) { this.production = production; this.kind = kind; @@ -120,7 +120,7 @@ static Symbol alt(Symbol[] symbols, String[] labels) { static Symbol error(String e) { return new ErrorAction(e); } - + /** * A convenience method to construct a ResolvingAction. * @param w The writer symbol @@ -129,32 +129,32 @@ static Symbol error(String e) { static Symbol resolve(Symbol w, Symbol r) { return new ResolvingAction(w, r); } - + private static class Fixup { public final Symbol[] symbols; public final int pos; - + public Fixup(Symbol[] symbols, int pos) { this.symbols = symbols; this.pos = pos; } } - + public Symbol flatten(Map map, Map> map2) { return this; } - + public int flattenedSize() { return 1; } - + /** * Flattens the given sub-array of symbols into an sub-array of symbols. Every * Sequence in the input are replaced by its production recursively. * Non-Sequence symbols, they internally have other symbols * those internal symbols also get flattened. - * + * * The algorithm does a few tricks to handle recursive symbol definitions. * In order to avoid infinite recursion with recursive symbols, we have a map * of Symbol->Symbol. Before fully constructing a flattened symbol for a @@ -168,7 +168,7 @@ public int flattenedSize() { * has not not be fully constructed yet, we copy a bunch of nulls. * Fix-up remembers all those null patches. The fix-ups gets finally * filled when we know the symbols to occupy those patches. - * + * * @param in The array of input symbols to flatten * @param start The position where the input sub-array starts. * @param out The output that receives the flattened list of symbols. The @@ -246,7 +246,7 @@ public Terminal(String printName) { public static class ImplicitAction extends Symbol { /** - * Set to true if and only if this implicit action is + * Set to true if and only if this implicit action is * a trailing action. That is, it is an action that follows * real symbol. E.g {@link Symbol#DEFAULT_END_ACTION}. */ @@ -255,13 +255,13 @@ public static class ImplicitAction extends Symbol { private ImplicitAction() { this(false); } - + private ImplicitAction(boolean isTrailing) { super(Kind.IMPLICIT_ACTION); this.isTrailing = isTrailing; } } - + protected static class Root extends Symbol { private Root(Symbol... symbols) { super(Kind.ROOT, makeProduction(symbols)); @@ -276,7 +276,7 @@ private static Symbol[] makeProduction(Symbol[] symbols) { return result; } } - + protected static class Sequence extends Symbol implements Iterable { private Sequence(Symbol[] productions) { super(Kind.SEQUENCE, productions); @@ -285,19 +285,19 @@ private Sequence(Symbol[] productions) { public Symbol get(int index) { return production[index]; } - + public int size() { return production.length; } - + public Iterator iterator() { return new Iterator() { private int pos = production.length; - + public boolean hasNext() { return 0 < pos; } - + public Symbol next() { if (0 < pos) { return production[--pos]; @@ -305,7 +305,7 @@ public Symbol next() { throw new NoSuchElementException(); } } - + public void remove() { throw new UnsupportedOperationException(); } @@ -320,7 +320,7 @@ public Sequence flatten(Map map, map.put(this, result); List l = new ArrayList(); map2.put(result, l); - + flatten(production, 0, result.production, 0, map, map2); for (Fixup f : l) { @@ -340,19 +340,19 @@ public final int flattenedSize() { public static class Repeater extends Symbol { public final Symbol end; - + private Repeater(Symbol end, Symbol... sequenceToRepeat) { super(Kind.REPEATER, makeProduction(sequenceToRepeat)); this.end = end; production[0] = this; } - + private static Symbol[] makeProduction(Symbol[] p) { Symbol[] result = new Symbol[p.length + 1]; System.arraycopy(p, 0, result, 1, p.length); return result; } - + @Override public Repeater flatten(Map map, Map> map2) { @@ -363,9 +363,9 @@ public Repeater flatten(Map map, } } - + /** - * Returns true if the Parser contains any Error symbol, indicating that it may fail + * Returns true if the Parser contains any Error symbol, indicating that it may fail * for some inputs. */ public static boolean hasErrors(Symbol symbol) { @@ -388,7 +388,7 @@ public static boolean hasErrors(Symbol symbol) { throw new RuntimeException("unknown symbol kind: " + symbol.kind); } } - + private static boolean hasErrors(Symbol root, Symbol[] symbols) { if(null != symbols) { for(Symbol s: symbols) { @@ -402,7 +402,7 @@ private static boolean hasErrors(Symbol root, Symbol[] symbols) { } return false; } - + public static class Alternative extends Symbol { public final Symbol[] symbols; public final String[] labels; @@ -411,15 +411,15 @@ private Alternative(Symbol[] symbols, String[] labels) { this.symbols = symbols; this.labels = labels; } - + public Symbol getSymbol(int index) { return symbols[index]; } - + public String getLabel(int index) { return labels[index]; } - + public int size() { return symbols.length; } @@ -468,7 +468,7 @@ public static class IntCheckAction extends Symbol { public static EnumAdjustAction enumAdjustAction(int rsymCount, Object[] adj) { return new EnumAdjustAction(rsymCount, adj); } - + public static class EnumAdjustAction extends IntCheckAction { public final Object[] adjustments; @Deprecated public EnumAdjustAction(int rsymCount, Object[] adjustments) { @@ -492,7 +492,7 @@ private ResolvingAction(Symbol writer, Symbol reader) { this.writer = writer; this.reader = reader; } - + @Override public ResolvingAction flatten(Map map, Map> map2) { @@ -501,7 +501,7 @@ public ResolvingAction flatten(Map map, } } - + public static SkipAction skipAction(Symbol symToSkip) { return new SkipAction(symToSkip); } @@ -512,7 +512,7 @@ public static class SkipAction extends ImplicitAction { super(true); this.symToSkip = symToSkip; } - + @Override public SkipAction flatten(Map map, Map> map2) { @@ -524,7 +524,7 @@ public SkipAction flatten(Map map, public static FieldAdjustAction fieldAdjustAction(int rindex, String fname) { return new FieldAdjustAction(rindex, fname); } - + public static class FieldAdjustAction extends ImplicitAction { public final int rindex; public final String fname; @@ -533,7 +533,7 @@ public static class FieldAdjustAction extends ImplicitAction { this.fname = fname; } } - + public static FieldOrderAction fieldOrderAction(Schema.Field[] fields) { return new FieldOrderAction(fields); } @@ -567,13 +567,13 @@ public static class UnionAdjustAction extends ImplicitAction { this.rindex = rindex; this.symToParse = symToParse; } - + @Override public UnionAdjustAction flatten(Map map, Map> map2) { return new UnionAdjustAction(rindex, symToParse.flatten(map, map2)); } - + } /** For JSON. */ @@ -587,11 +587,11 @@ public static class EnumLabelsAction extends IntCheckAction { super(symbols.size()); this.symbols = symbols; } - + public String getLabel(int n) { return symbols.get(n); } - + public int findLabel(String l) { if (l != null) { for (int i = 0; i < symbols.size(); i++) { @@ -633,8 +633,9 @@ public int findLabel(String l) { public static final Symbol RECORD_END = new ImplicitAction(true); public static final Symbol UNION_END = new ImplicitAction(true); public static final Symbol FIELD_END = new ImplicitAction(true); - + public static final Symbol DEFAULT_END_ACTION = new ImplicitAction(true); public static final Symbol MAP_KEY_MARKER = new Symbol.Terminal("map-key-marker"); } + From 1158152de6debcf8e5a532585a68a39ab64d78ed Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Mon, 4 Apr 2016 11:33:13 -0400 Subject: [PATCH 06/22] [fix] remove @author from javadoc, add some useful comment instead, reformat file to 2 space indentation --- .../avro/compiler/idl/SchemaResolver.java | 357 +++++++++--------- 1 file changed, 176 insertions(+), 181 deletions(-) diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java index df0c7ac2ef5..9dabdd42006 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java @@ -28,202 +28,197 @@ import org.codehaus.jackson.JsonNode; /** - * - * @author zoly + * Utility class to resolve schemas that are unavailable at the time they are referenced in the IDL. */ + final class SchemaResolver { - private SchemaResolver() { } + private SchemaResolver() { + } - private static final String UR_SCHEMA_ATTR = "org.apache.avro.compiler.idl.unresolved.name"; + private static final String UR_SCHEMA_ATTR = "org.apache.avro.compiler.idl.unresolved.name"; - static Schema unresolvedSchema(final String name) { - Schema schema = Schema.createRecord("UnresolvedSchema", "unresolved schema", - "org.apache.avro.compiler", false, Collections.EMPTY_LIST); - schema.addProp(UR_SCHEMA_ATTR, name); - return schema; - } + static Schema unresolvedSchema(final String name) { + Schema schema = Schema.createRecord("UnresolvedSchema", "unresolved schema", + "org.apache.avro.compiler", false, Collections.EMPTY_LIST); + schema.addProp(UR_SCHEMA_ATTR, name); + return schema; + } - static boolean isUnresolvedSchema(final Schema schema) { - return (schema.getType() == Schema.Type.RECORD && schema.getProp(UR_SCHEMA_ATTR) != null); - } + static boolean isUnresolvedSchema(final Schema schema) { + return (schema.getType() == Schema.Type.RECORD && schema.getProp(UR_SCHEMA_ATTR) != null); + } - static String getUnresolvedSchemaName(final Schema schema) { - String name = schema.getProp(UR_SCHEMA_ATTR); - if (name == null) { - throw new IllegalArgumentException("Schema " + schema + " is not a unresolved schema"); - } else { - return name; - } + static String getUnresolvedSchemaName(final Schema schema) { + String name = schema.getProp(UR_SCHEMA_ATTR); + if (name == null) { + throw new IllegalArgumentException("Schema " + schema + " is not a unresolved schema"); + } else { + return name; } - - - - static Protocol resolve(final Protocol protocol) { - Protocol result = new Protocol(protocol.getName(), protocol.getDoc(), protocol.getNamespace()); - final Collection types = protocol.getTypes(); - List newSchemas = new ArrayList(types.size()); - Map processed = new HashMap(); - for (Schema schema : types) { - newSchemas.add(resolve(schema, protocol, processed)); - } - result.setTypes(newSchemas); - - for (Map.Entry entry : protocol.getMessages().entrySet()) { - Protocol.Message value = entry.getValue(); - Protocol.Message nvalue; - if (value.isOneWay()) { - Schema request = value.getRequest(); - nvalue = result.createMessage(value.getName(), value.getDoc(), - value.getObjectProps(), intern(request, processed)); - } else { - Schema request = value.getRequest(); - Schema response = value.getResponse(); - Schema errors = value.getErrors(); - nvalue = result.createMessage(value.getName(), value.getDoc(), - value.getObjectProps(), intern(request, processed), - intern(response, processed), intern(errors, processed)); - } - result.getMessages().put(entry.getKey(), nvalue); - } - copyProps(protocol, result); - return result; + } + + static Protocol resolve(final Protocol protocol) { + Protocol result = new Protocol(protocol.getName(), protocol.getDoc(), protocol.getNamespace()); + final Collection types = protocol.getTypes(); + List newSchemas = new ArrayList(types.size()); + Map processed = new HashMap(); + for (Schema schema : types) { + newSchemas.add(resolve(schema, protocol, processed)); } - - private static void copyProps(final JsonProperties from, final JsonProperties to) { - for (Map.Entry entry : from.getJsonProps().entrySet()) { - to.addProp(entry.getKey(), entry.getValue()); - } + result.setTypes(newSchemas); + + for (Map.Entry entry : protocol.getMessages().entrySet()) { + Protocol.Message value = entry.getValue(); + Protocol.Message nvalue; + if (value.isOneWay()) { + Schema request = value.getRequest(); + nvalue = result.createMessage(value.getName(), value.getDoc(), + value.getObjectProps(), intern(request, processed)); + } else { + Schema request = value.getRequest(); + Schema response = value.getResponse(); + Schema errors = value.getErrors(); + nvalue = result.createMessage(value.getName(), value.getDoc(), + value.getObjectProps(), intern(request, processed), + intern(response, processed), intern(errors, processed)); + } + result.getMessages().put(entry.getKey(), nvalue); } + copyProps(protocol, result); + return result; + } - - static Schema resolve(final Schema schema, final Protocol protocol, final Map processed) { - final String fullName = schema.getFullName(); - if (fullName != null && processed.containsKey(fullName)) { - return processed.get(schema.getFullName()); - } else if (isUnresolvedSchema(schema)) { - final String unresolvedSchemaName = getUnresolvedSchemaName(schema); - Schema type = protocol.getType(unresolvedSchemaName); - if (type == null) { - throw new IllegalArgumentException("Cannot resolve " + unresolvedSchemaName); - } - return resolve(type, protocol, processed); - } else { - switch (schema.getType()) { - case RECORD: - Schema createRecord = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), - schema.isError()); - processed.put(schema.getFullName(), createRecord); - final List currFields = schema.getFields(); - List newFields = new ArrayList(currFields.size()); - for (Schema.Field field : currFields) { - Schema.Field nf = new Schema.Field(field.name(), resolve(field.schema(), protocol, processed), - field.doc(), field.defaultVal(), field.order()); - for (String alias : field.aliases()) { - nf.addAlias(alias); - } - newFields.add(nf); - } - createRecord.setFields(newFields); - final LogicalType lt = schema.getLogicalType(); - if (lt != null) { - lt.addToSchema(createRecord); - } - copyProps(schema, createRecord); - return createRecord; - case MAP: - Schema result = Schema.createMap(resolve(schema.getValueType(), protocol, processed)); - copyProps(schema, result); - return result; - case ARRAY: - Schema aresult = Schema.createArray(resolve(schema.getElementType(), protocol, processed)); - copyProps(schema, aresult); - return aresult; - case UNION: - final List uTypes = schema.getTypes(); - List newTypes = new ArrayList(uTypes.size()); - for (Schema s : uTypes) { - newTypes.add(resolve(s, protocol, processed)); - } - Schema bresult = Schema.createUnion(newTypes); - copyProps(schema, bresult); - return bresult; - case ENUM: - case FIXED: - case STRING: - case BYTES: - case INT: - case LONG: - case FLOAT: - case DOUBLE: - case BOOLEAN: - case NULL: - return schema; - default: - throw new RuntimeException("Unknown type: " + schema); + private static void copyProps(final JsonProperties from, final JsonProperties to) { + for (Map.Entry entry : from.getJsonProps().entrySet()) { + to.addProp(entry.getKey(), entry.getValue()); + } + } + + static Schema resolve(final Schema schema, final Protocol protocol, final Map processed) { + final String fullName = schema.getFullName(); + if (fullName != null && processed.containsKey(fullName)) { + return processed.get(schema.getFullName()); + } else if (isUnresolvedSchema(schema)) { + final String unresolvedSchemaName = getUnresolvedSchemaName(schema); + Schema type = protocol.getType(unresolvedSchemaName); + if (type == null) { + throw new IllegalArgumentException("Cannot resolve " + unresolvedSchemaName); + } + return resolve(type, protocol, processed); + } else { + switch (schema.getType()) { + case RECORD: + Schema createRecord = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), + schema.isError()); + processed.put(schema.getFullName(), createRecord); + final List currFields = schema.getFields(); + List newFields = new ArrayList(currFields.size()); + for (Schema.Field field : currFields) { + Schema.Field nf = new Schema.Field(field.name(), resolve(field.schema(), protocol, processed), + field.doc(), field.defaultVal(), field.order()); + for (String alias : field.aliases()) { + nf.addAlias(alias); } - } + newFields.add(nf); + } + createRecord.setFields(newFields); + final LogicalType lt = schema.getLogicalType(); + if (lt != null) { + lt.addToSchema(createRecord); + } + copyProps(schema, createRecord); + return createRecord; + case MAP: + Schema result = Schema.createMap(resolve(schema.getValueType(), protocol, processed)); + copyProps(schema, result); + return result; + case ARRAY: + Schema aresult = Schema.createArray(resolve(schema.getElementType(), protocol, processed)); + copyProps(schema, aresult); + return aresult; + case UNION: + final List uTypes = schema.getTypes(); + List newTypes = new ArrayList(uTypes.size()); + for (Schema s : uTypes) { + newTypes.add(resolve(s, protocol, processed)); + } + Schema bresult = Schema.createUnion(newTypes); + copyProps(schema, bresult); + return bresult; + case ENUM: + case FIXED: + case STRING: + case BYTES: + case INT: + case LONG: + case FLOAT: + case DOUBLE: + case BOOLEAN: + case NULL: + return schema; + default: + throw new RuntimeException("Unknown type: " + schema); + } } + } - public static Schema intern(final Schema schema, final Map processed) { - if (schema == null) { - return null; - } - final String fullName = schema.getFullName(); - if (fullName != null && processed.containsKey(fullName)) { - return processed.get(schema.getFullName()); - } else { - switch (schema.getType()) { - case RECORD: - Schema createRecord = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), - schema.isError()); - processed.put(schema.getFullName(), createRecord); - final List currFields = schema.getFields(); - List newFields = new ArrayList(currFields.size()); - for (Schema.Field field : currFields) { - Schema.Field nf = new Schema.Field(field.name(), intern(field.schema(), processed), - field.doc(), field.defaultVal(), field.order()); - for (String alias : field.aliases()) { - nf.addAlias(alias); - } - newFields.add(nf); - } - createRecord.setFields(newFields); - final LogicalType lt = schema.getLogicalType(); - if (lt != null) { - lt.addToSchema(createRecord); - } - copyProps(schema, createRecord); - return createRecord; - case MAP: - return Schema.createMap(intern(schema.getValueType(), processed)); - case ARRAY: - return Schema.createArray(intern(schema.getElementType(), processed)); - case UNION: - final List uTypes = schema.getTypes(); - List newTypes = new ArrayList(uTypes.size()); - for (Schema s : uTypes) { - newTypes.add(intern(s, processed)); - } - return Schema.createUnion(newTypes); - case ENUM: - case FIXED: - case STRING: - case BYTES: - case INT: - case LONG: - case FLOAT: - case DOUBLE: - case BOOLEAN: - case NULL: - return schema; - default: - throw new RuntimeException("Unknown type: " + schema); + public static Schema intern(final Schema schema, final Map processed) { + if (schema == null) { + return null; + } + final String fullName = schema.getFullName(); + if (fullName != null && processed.containsKey(fullName)) { + return processed.get(schema.getFullName()); + } else { + switch (schema.getType()) { + case RECORD: + Schema createRecord = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), + schema.isError()); + processed.put(schema.getFullName(), createRecord); + final List currFields = schema.getFields(); + List newFields = new ArrayList(currFields.size()); + for (Schema.Field field : currFields) { + Schema.Field nf = new Schema.Field(field.name(), intern(field.schema(), processed), + field.doc(), field.defaultVal(), field.order()); + for (String alias : field.aliases()) { + nf.addAlias(alias); } - } + newFields.add(nf); + } + createRecord.setFields(newFields); + final LogicalType lt = schema.getLogicalType(); + if (lt != null) { + lt.addToSchema(createRecord); + } + copyProps(schema, createRecord); + return createRecord; + case MAP: + return Schema.createMap(intern(schema.getValueType(), processed)); + case ARRAY: + return Schema.createArray(intern(schema.getElementType(), processed)); + case UNION: + final List uTypes = schema.getTypes(); + List newTypes = new ArrayList(uTypes.size()); + for (Schema s : uTypes) { + newTypes.add(intern(s, processed)); + } + return Schema.createUnion(newTypes); + case ENUM: + case FIXED: + case STRING: + case BYTES: + case INT: + case LONG: + case FLOAT: + case DOUBLE: + case BOOLEAN: + case NULL: + return schema; + default: + throw new RuntimeException("Unknown type: " + schema); + } } - - - + } } From b5e13ff04d1e9b6e5bb02b482b6ef0111268a9ed Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Mon, 4 Apr 2016 11:46:10 -0400 Subject: [PATCH 07/22] [cleanup] --- lang/java/ipc/nb-configuration.xml | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 lang/java/ipc/nb-configuration.xml diff --git a/lang/java/ipc/nb-configuration.xml b/lang/java/ipc/nb-configuration.xml deleted file mode 100644 index 93af9a7ce41..00000000000 --- a/lang/java/ipc/nb-configuration.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - JDK_1.6 - - From df57d3ad29918b6c45b6fce15d4f0815024d49f2 Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Mon, 4 Apr 2016 11:50:52 -0400 Subject: [PATCH 08/22] [fix] use a logger instead of System.out + reformat to 2 space indentation --- .../apache/avro/compiler/idl/TestCycle.java | 102 +++++++++--------- 1 file changed, 52 insertions(+), 50 deletions(-) diff --git a/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestCycle.java b/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestCycle.java index a38e978ef7d..bc7695956e1 100644 --- a/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestCycle.java +++ b/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestCycle.java @@ -23,7 +23,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; -import junit.framework.Assert; +import org.junit.Assert; import org.apache.avro.Protocol; import org.apache.avro.Schema; import org.apache.avro.compiler.specific.SpecificCompiler; @@ -36,6 +36,8 @@ import org.apache.avro.io.DecoderFactory; import org.apache.avro.io.EncoderFactory; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @@ -43,63 +45,63 @@ */ public class TestCycle { - @Test - public void testCycleGeneration() throws ParseException, IOException { - final ClassLoader cl = Thread.currentThread().getContextClassLoader(); - Idl idl = new Idl(cl.getResourceAsStream("input/cycle.avdl"), - "UTF-8"); - Protocol protocol = idl.CompilationUnit(); - String json = protocol.toString(); - System.out.println(json); + private static final Logger LOG = LoggerFactory.getLogger(TestCycle.class); + + @Test + public void testCycleGeneration() throws ParseException, IOException { + final ClassLoader cl = Thread.currentThread().getContextClassLoader(); + Idl idl = new Idl(cl.getResourceAsStream("input/cycle.avdl"), + "UTF-8"); + Protocol protocol = idl.CompilationUnit(); + String json = protocol.toString(); + LOG.info(json); - SpecificCompiler compiler = new SpecificCompiler(protocol); - compiler.setStringType(GenericData.StringType.String); - File output = new File("./target"); - compiler.compileToDestination(null, output); + SpecificCompiler compiler = new SpecificCompiler(protocol); + compiler.setStringType(GenericData.StringType.String); + File output = new File("./target"); + compiler.compileToDestination(null, output); - Map schemas = new HashMap(); - for (Schema schema : protocol.getTypes()) { - final String name = schema.getName(); - schemas.put(name, schema); - } - - GenericRecordBuilder rb2 = new GenericRecordBuilder(schemas.get("SampleNode")); - rb2.set("count", 10); - rb2.set("subNodes", Collections.EMPTY_LIST); - GenericData.Record node = rb2.build(); + Map schemas = new HashMap(); + for (Schema schema : protocol.getTypes()) { + final String name = schema.getName(); + schemas.put(name, schema); + } - GenericRecordBuilder mb = new GenericRecordBuilder(schemas.get("Method")); - mb.set("declaringClass", "Test"); - mb.set("methodName", "test"); - GenericData.Record method = mb.build(); + GenericRecordBuilder rb2 = new GenericRecordBuilder(schemas.get("SampleNode")); + rb2.set("count", 10); + rb2.set("subNodes", Collections.EMPTY_LIST); + GenericData.Record node = rb2.build(); - GenericRecordBuilder spb = new GenericRecordBuilder(schemas.get("SamplePair")); - spb.set("method", method); - spb.set("node", node); - GenericData.Record sp = spb.build(); + GenericRecordBuilder mb = new GenericRecordBuilder(schemas.get("Method")); + mb.set("declaringClass", "Test"); + mb.set("methodName", "test"); + GenericData.Record method = mb.build(); + GenericRecordBuilder spb = new GenericRecordBuilder(schemas.get("SamplePair")); + spb.set("method", method); + spb.set("node", node); + GenericData.Record sp = spb.build(); - GenericRecordBuilder rb = new GenericRecordBuilder(schemas.get("SampleNode")); - rb.set("count", 10); - rb.set("subNodes", Arrays.asList(sp)); - GenericData.Record record = rb.build(); + GenericRecordBuilder rb = new GenericRecordBuilder(schemas.get("SampleNode")); + rb.set("count", 10); + rb.set("subNodes", Arrays.asList(sp)); + GenericData.Record record = rb.build(); - serDeserRecord(record); + serDeserRecord(record); - } - - private static void serDeserRecord(GenericData.Record data) throws IOException { - ByteArrayOutputStream bab = new ByteArrayOutputStream(); - GenericDatumWriter writer = new GenericDatumWriter(data.getSchema()); - final BinaryEncoder directBinaryEncoder = EncoderFactory.get().directBinaryEncoder(bab, null); - writer.write(data, directBinaryEncoder); - directBinaryEncoder.flush(); - ByteArrayInputStream bis = new ByteArrayInputStream(bab.toByteArray(), 0, bab.size()); - GenericDatumReader reader = new GenericDatumReader(data.getSchema()); - BinaryDecoder directBinaryDecoder = DecoderFactory.get().directBinaryDecoder(bis, null); - GenericData.Record read = (GenericData.Record) reader.read(null, directBinaryDecoder); - Assert.assertEquals(data.toString(), read.toString()); - } + } + private static void serDeserRecord(GenericData.Record data) throws IOException { + ByteArrayOutputStream bab = new ByteArrayOutputStream(); + GenericDatumWriter writer = new GenericDatumWriter(data.getSchema()); + final BinaryEncoder directBinaryEncoder = EncoderFactory.get().directBinaryEncoder(bab, null); + writer.write(data, directBinaryEncoder); + directBinaryEncoder.flush(); + ByteArrayInputStream bis = new ByteArrayInputStream(bab.toByteArray(), 0, bab.size()); + GenericDatumReader reader = new GenericDatumReader(data.getSchema()); + BinaryDecoder directBinaryDecoder = DecoderFactory.get().directBinaryDecoder(bis, null); + GenericData.Record read = (GenericData.Record) reader.read(null, directBinaryDecoder); + Assert.assertEquals(data.toString(), read.toString()); + } } From a084fcdb5e79b1b8b86710f836ac6afce08c4a54 Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Mon, 4 Apr 2016 11:52:08 -0400 Subject: [PATCH 09/22] [fix] indentation --- .../main/java/org/apache/avro/compiler/idl/SchemaResolver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java index 9dabdd42006..6e849c0c22b 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java @@ -174,7 +174,7 @@ public static Schema intern(final Schema schema, final Map proce switch (schema.getType()) { case RECORD: Schema createRecord = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), - schema.isError()); + schema.isError()); processed.put(schema.getFullName(), createRecord); final List currFields = schema.getFields(); List newFields = new ArrayList(currFields.size()); From a593cb03f93a9d449723c0e5d765bf2dcf55e132 Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Sun, 17 Apr 2016 18:15:57 -0400 Subject: [PATCH 10/22] [cleanup] eliminate commented code. --- .../compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj | 1 - 1 file changed, 1 deletion(-) diff --git a/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj b/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj index a8f06234625..13da4badc69 100644 --- a/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj +++ b/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj @@ -1465,7 +1465,6 @@ Schema ReferenceType(): name = namespace + "." + name; Schema type = names.get(name); if (type == null) - //throw error("Undefined name '" + name + "'", token); { type = SchemaResolver.unresolvedSchema(name); } From cc224aba71ae32f7f371139701bf8f1a9587ad38 Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Sun, 17 Apr 2016 21:54:49 -0400 Subject: [PATCH 11/22] [add] some comments --- .../avro/compiler/idl/SchemaResolver.java | 65 ++++++++++++------- 1 file changed, 42 insertions(+), 23 deletions(-) diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java index 6e849c0c22b..6d897cafef9 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java @@ -58,15 +58,20 @@ static String getUnresolvedSchemaName(final Schema schema) { } } + /** + * Resolve all unresolved schema references from a protocol. + * @param protocol - the protocol with unresolved schema references. + * @return - a new protocol instance based on the provided protocol with all unresolved schema references resolved. + */ static Protocol resolve(final Protocol protocol) { Protocol result = new Protocol(protocol.getName(), protocol.getDoc(), protocol.getNamespace()); final Collection types = protocol.getTypes(); List newSchemas = new ArrayList(types.size()); - Map processed = new HashMap(); + Map resolved = new HashMap(); for (Schema schema : types) { - newSchemas.add(resolve(schema, protocol, processed)); + newSchemas.add(resolve(schema, protocol, resolved)); } - result.setTypes(newSchemas); + result.setTypes(newSchemas); // replace types with resolved ones for (Map.Entry entry : protocol.getMessages().entrySet()) { Protocol.Message value = entry.getValue(); @@ -74,14 +79,14 @@ static Protocol resolve(final Protocol protocol) { if (value.isOneWay()) { Schema request = value.getRequest(); nvalue = result.createMessage(value.getName(), value.getDoc(), - value.getObjectProps(), intern(request, processed)); + value.getObjectProps(), getResolvedSchema(request, resolved)); } else { Schema request = value.getRequest(); Schema response = value.getResponse(); Schema errors = value.getErrors(); nvalue = result.createMessage(value.getName(), value.getDoc(), - value.getObjectProps(), intern(request, processed), - intern(response, processed), intern(errors, processed)); + value.getObjectProps(), getResolvedSchema(request, resolved), + getResolvedSchema(response, resolved), getResolvedSchema(errors, resolved)); } result.getMessages().put(entry.getKey(), nvalue); } @@ -95,27 +100,35 @@ private static void copyProps(final JsonProperties from, final JsonProperties to } } - static Schema resolve(final Schema schema, final Protocol protocol, final Map processed) { + /** + * Resolve all unresolved schema references. + * @param schema - the schema to resolved references for. + * @param protocol - the protocol we resolve the schema's for. + * (we lookup all unresolved schema references in the protocol) + * @param resolved - a map of all resolved schema's so far. + * @return - a instance of the resolved schema. + */ + static Schema resolve(final Schema schema, final Protocol protocol, final Map resolved) { final String fullName = schema.getFullName(); - if (fullName != null && processed.containsKey(fullName)) { - return processed.get(schema.getFullName()); + if (fullName != null && resolved.containsKey(fullName)) { + return resolved.get(schema.getFullName()); } else if (isUnresolvedSchema(schema)) { final String unresolvedSchemaName = getUnresolvedSchemaName(schema); Schema type = protocol.getType(unresolvedSchemaName); if (type == null) { throw new IllegalArgumentException("Cannot resolve " + unresolvedSchemaName); } - return resolve(type, protocol, processed); + return resolve(type, protocol, resolved); } else { switch (schema.getType()) { case RECORD: Schema createRecord = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.isError()); - processed.put(schema.getFullName(), createRecord); + resolved.put(schema.getFullName(), createRecord); final List currFields = schema.getFields(); List newFields = new ArrayList(currFields.size()); for (Schema.Field field : currFields) { - Schema.Field nf = new Schema.Field(field.name(), resolve(field.schema(), protocol, processed), + Schema.Field nf = new Schema.Field(field.name(), resolve(field.schema(), protocol, resolved), field.doc(), field.defaultVal(), field.order()); for (String alias : field.aliases()) { nf.addAlias(alias); @@ -130,18 +143,18 @@ static Schema resolve(final Schema schema, final Protocol protocol, final Map uTypes = schema.getTypes(); List newTypes = new ArrayList(uTypes.size()); for (Schema s : uTypes) { - newTypes.add(resolve(s, protocol, processed)); + newTypes.add(resolve(s, protocol, resolved)); } Schema bresult = Schema.createUnion(newTypes); copyProps(schema, bresult); @@ -163,23 +176,29 @@ static Schema resolve(final Schema schema, final Protocol protocol, final Map processed) { + /** + * get the resolved schema. + * @param schema - the schema we want to get the resolved equivalent for. + * @param resolved - a Map wil all resolved schemas + * @return - the resolved schema. + */ + public static Schema getResolvedSchema(final Schema schema, final Map resolved) { if (schema == null) { return null; } final String fullName = schema.getFullName(); - if (fullName != null && processed.containsKey(fullName)) { - return processed.get(schema.getFullName()); + if (fullName != null && resolved.containsKey(fullName)) { + return resolved.get(schema.getFullName()); } else { switch (schema.getType()) { case RECORD: Schema createRecord = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.isError()); - processed.put(schema.getFullName(), createRecord); + resolved.put(schema.getFullName(), createRecord); final List currFields = schema.getFields(); List newFields = new ArrayList(currFields.size()); for (Schema.Field field : currFields) { - Schema.Field nf = new Schema.Field(field.name(), intern(field.schema(), processed), + Schema.Field nf = new Schema.Field(field.name(), getResolvedSchema(field.schema(), resolved), field.doc(), field.defaultVal(), field.order()); for (String alias : field.aliases()) { nf.addAlias(alias); @@ -194,14 +213,14 @@ public static Schema intern(final Schema schema, final Map proce copyProps(schema, createRecord); return createRecord; case MAP: - return Schema.createMap(intern(schema.getValueType(), processed)); + return Schema.createMap(getResolvedSchema(schema.getValueType(), resolved)); case ARRAY: - return Schema.createArray(intern(schema.getElementType(), processed)); + return Schema.createArray(getResolvedSchema(schema.getElementType(), resolved)); case UNION: final List uTypes = schema.getTypes(); List newTypes = new ArrayList(uTypes.size()); for (Schema s : uTypes) { - newTypes.add(intern(s, processed)); + newTypes.add(getResolvedSchema(s, resolved)); } return Schema.createUnion(newTypes); case ENUM: From 21efba087b9fec37500b59e21a8bf5be546ea5a8 Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Sun, 17 Apr 2016 23:30:10 -0400 Subject: [PATCH 12/22] [add] better exception messaging. --- .../apache/avro/compiler/idl/SchemaResolver.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java index 6d897cafef9..56386b3dbb7 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java @@ -38,9 +38,15 @@ private SchemaResolver() { private static final String UR_SCHEMA_ATTR = "org.apache.avro.compiler.idl.unresolved.name"; + private static final String UR_SCHEMA_NAME = "UnresolvedSchema"; + + private static final String UR_SCHEMA_NS = "org.apache.avro.compiler"; + static Schema unresolvedSchema(final String name) { - Schema schema = Schema.createRecord("UnresolvedSchema", "unresolved schema", - "org.apache.avro.compiler", false, Collections.EMPTY_LIST); + + + Schema schema = Schema.createRecord(UR_SCHEMA_NAME, "unresolved schema", + UR_SCHEMA_NS, false, Collections.EMPTY_LIST); schema.addProp(UR_SCHEMA_ATTR, name); return schema; } @@ -50,9 +56,13 @@ static boolean isUnresolvedSchema(final Schema schema) { } static String getUnresolvedSchemaName(final Schema schema) { + if (schema.getType() != Schema.Type.RECORD || !UR_SCHEMA_NAME.equals(schema.getName()) + || !UR_SCHEMA_NS.equals(schema.getNamespace())) { + throw new IllegalArgumentException("Not a unresolved schema: " + schema); + } String name = schema.getProp(UR_SCHEMA_ATTR); if (name == null) { - throw new IllegalArgumentException("Schema " + schema + " is not a unresolved schema"); + throw new IllegalArgumentException("Schema " + schema + " must have attribute: " + UR_SCHEMA_ATTR); } else { return name; } From c3964345189581db8347bf0e038a0559d7a7626c Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Mon, 11 Jul 2016 11:28:23 -0400 Subject: [PATCH 13/22] [add] add more to gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 372789a7208..03887815ab0 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ target /build test-output /dist +/lang/java/compiler/nbactions.xml +/lang/java/compiler/nb-configuration.xml \ No newline at end of file From c3384e4d016bac264f6577fff9543199a1fb1970 Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Tue, 7 Feb 2017 13:36:56 -0500 Subject: [PATCH 14/22] [add] incomplete implementation to use generic schema traverser. --- .../test/java/org/apache/avro/TestFixed.java | 34 +++ lang/java/compiler/pom.xml | 11 +- .../avro/compiler/idl/SchemaResolver.java | 76 +++--- .../avro/compiler/schema/SchemaVisitor.java | 67 +++++ .../compiler/schema/SchemaVisitorAction.java | 44 ++++ .../apache/avro/compiler/schema/Schemas.java | 245 ++++++++++++++++++ 6 files changed, 432 insertions(+), 45 deletions(-) create mode 100644 lang/java/avro/src/test/java/org/apache/avro/TestFixed.java create mode 100644 lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/SchemaVisitor.java create mode 100644 lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/SchemaVisitorAction.java create mode 100644 lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/Schemas.java diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestFixed.java b/lang/java/avro/src/test/java/org/apache/avro/TestFixed.java new file mode 100644 index 00000000000..14ff5cef49a --- /dev/null +++ b/lang/java/avro/src/test/java/org/apache/avro/TestFixed.java @@ -0,0 +1,34 @@ +/* + * Copyright 2017 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro; + +import java.nio.ByteBuffer; +import org.junit.Assert; +import org.junit.Test; + +public class TestFixed { + + + @Test + public void testFixedDefaultValueDrop() { + Schema md5 = SchemaBuilder.builder().fixed("MD5").size(16); + Schema frec = SchemaBuilder.builder().record("test") + .fields().name("hash").type(md5).withDefault(ByteBuffer.wrap(new byte[16])).endRecord(); + Schema.Field field = frec.getField("hash"); + Assert.assertNotNull(field.defaultVal()); + } + +} diff --git a/lang/java/compiler/pom.xml b/lang/java/compiler/pom.xml index 3ff010197f7..2a35c456bbf 100644 --- a/lang/java/compiler/pom.xml +++ b/lang/java/compiler/pom.xml @@ -138,12 +138,11 @@ guava ${guava.version} test - - - com.google.code.findbugs - jsr305 - - + + + com.google.code.findbugs + jsr305 + 3.0.1 diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java index 56386b3dbb7..6aed7878283 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java @@ -21,11 +21,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.avro.JsonProperties; -import org.apache.avro.LogicalType; import org.apache.avro.Protocol; import org.apache.avro.Schema; -import org.codehaus.jackson.JsonNode; +import org.apache.avro.compiler.schema.Schemas; /** * Utility class to resolve schemas that are unavailable at the time they are referenced in the IDL. @@ -39,12 +37,12 @@ private SchemaResolver() { private static final String UR_SCHEMA_ATTR = "org.apache.avro.compiler.idl.unresolved.name"; private static final String UR_SCHEMA_NAME = "UnresolvedSchema"; - - private static final String UR_SCHEMA_NS = "org.apache.avro.compiler"; - + + private static final String UR_SCHEMA_NS = "org.apache.avro.compiler"; + static Schema unresolvedSchema(final String name) { - - + + Schema schema = Schema.createRecord(UR_SCHEMA_NAME, "unresolved schema", UR_SCHEMA_NS, false, Collections.EMPTY_LIST); schema.addProp(UR_SCHEMA_ATTR, name); @@ -52,12 +50,13 @@ static Schema unresolvedSchema(final String name) { } static boolean isUnresolvedSchema(final Schema schema) { - return (schema.getType() == Schema.Type.RECORD && schema.getProp(UR_SCHEMA_ATTR) != null); + return (schema.getType() == Schema.Type.RECORD && schema.getProp(UR_SCHEMA_ATTR) != null + && UR_SCHEMA_NAME.equals(schema.getName()) + && UR_SCHEMA_NS.equals(schema.getNamespace())); } static String getUnresolvedSchemaName(final Schema schema) { - if (schema.getType() != Schema.Type.RECORD || !UR_SCHEMA_NAME.equals(schema.getName()) - || !UR_SCHEMA_NS.equals(schema.getNamespace())) { + if (!isUnresolvedSchema(schema)) { throw new IllegalArgumentException("Not a unresolved schema: " + schema); } String name = schema.getProp(UR_SCHEMA_ATTR); @@ -100,15 +99,10 @@ static Protocol resolve(final Protocol protocol) { } result.getMessages().put(entry.getKey(), nvalue); } - copyProps(protocol, result); + Schemas.copyProperties(protocol, result); return result; } - private static void copyProps(final JsonProperties from, final JsonProperties to) { - for (Map.Entry entry : from.getJsonProps().entrySet()) { - to.addProp(entry.getKey(), entry.getValue()); - } - } /** * Resolve all unresolved schema references. @@ -138,27 +132,26 @@ static Schema resolve(final Schema schema, final Protocol protocol, final Map currFields = schema.getFields(); List newFields = new ArrayList(currFields.size()); for (Schema.Field field : currFields) { + if (field.name().equals("hash")) { + System.err.println(field); + } Schema.Field nf = new Schema.Field(field.name(), resolve(field.schema(), protocol, resolved), field.doc(), field.defaultVal(), field.order()); - for (String alias : field.aliases()) { - nf.addAlias(alias); - } + Schemas.copyAliases(field, nf); + Schemas.copyProperties(field, nf); newFields.add(nf); } createRecord.setFields(newFields); - final LogicalType lt = schema.getLogicalType(); - if (lt != null) { - lt.addToSchema(createRecord); - } - copyProps(schema, createRecord); + Schemas.copyLogicalTypes(schema, createRecord); + Schemas.copyProperties(schema, createRecord); return createRecord; case MAP: Schema result = Schema.createMap(resolve(schema.getValueType(), protocol, resolved)); - copyProps(schema, result); + Schemas.copyProperties(schema, result); return result; case ARRAY: Schema aresult = Schema.createArray(resolve(schema.getElementType(), protocol, resolved)); - copyProps(schema, aresult); + Schemas.copyProperties(schema, aresult); return aresult; case UNION: final List uTypes = schema.getTypes(); @@ -167,7 +160,7 @@ static Schema resolve(final Schema schema, final Protocol protocol, final Map currFields = schema.getFields(); List newFields = new ArrayList(currFields.size()); for (Schema.Field field : currFields) { + if (field.name().equals("hash")) { + System.err.println(field); + } Schema.Field nf = new Schema.Field(field.name(), getResolvedSchema(field.schema(), resolved), field.doc(), field.defaultVal(), field.order()); - for (String alias : field.aliases()) { - nf.addAlias(alias); - } + Schemas.copyAliases(field, nf); + Schemas.copyProperties(field, nf); newFields.add(nf); } createRecord.setFields(newFields); - final LogicalType lt = schema.getLogicalType(); - if (lt != null) { - lt.addToSchema(createRecord); - } - copyProps(schema, createRecord); + Schemas.copyLogicalTypes(schema, createRecord); + Schemas.copyProperties(schema, createRecord); return createRecord; case MAP: - return Schema.createMap(getResolvedSchema(schema.getValueType(), resolved)); + Schema createMap = Schema.createMap(getResolvedSchema(schema.getValueType(), resolved)); + Schemas.copyProperties(schema, createMap); + return createMap; case ARRAY: - return Schema.createArray(getResolvedSchema(schema.getElementType(), resolved)); + Schema createArray = Schema.createArray(getResolvedSchema(schema.getElementType(), resolved)); + Schemas.copyProperties(schema, createArray); + return createArray; case UNION: final List uTypes = schema.getTypes(); List newTypes = new ArrayList(uTypes.size()); for (Schema s : uTypes) { newTypes.add(getResolvedSchema(s, resolved)); } - return Schema.createUnion(newTypes); + Schema createUnion = Schema.createUnion(newTypes); + Schemas.copyProperties(schema, createUnion); + return createUnion; case ENUM: case FIXED: case STRING: diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/SchemaVisitor.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/SchemaVisitor.java new file mode 100644 index 00000000000..df667f46b66 --- /dev/null +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/SchemaVisitor.java @@ -0,0 +1,67 @@ + /* + * Copyright (c) 2001 - 2016, Zoltan Farkas All Rights Reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package org.apache.avro.compiler.schema; + +import javax.annotation.CheckReturnValue; +import javax.annotation.Nonnull; +import javax.annotation.ParametersAreNonnullByDefault; +import org.apache.avro.Schema; + +/** + * @author zoly + */ +@ParametersAreNonnullByDefault +public interface SchemaVisitor { + + /** + * Invoked for schemas that do not have "child" schemas (like string, int ...) + * or for a previously encountered schema with children, + * which will be treated as a terminal. (to avoid circular recursion) + * @param terminal + * @return + */ + @Nonnull + @CheckReturnValue + SchemaVisitorAction visitTerminal(Schema terminal); + + /** + * Invoked for schema with children before proceeding to visit the children. + * @param nonTerminal + * @return + */ + @Nonnull + @CheckReturnValue + SchemaVisitorAction visitNonTerminal(Schema nonTerminal); + + /** + * Invoked for schemas with children after its children have been visited. + * @param nonTerminal + * @return + */ + @Nonnull + @CheckReturnValue + SchemaVisitorAction afterVisitNonTerminal(Schema nonTerminal); + + + /** + * Invoked when visiting is complete. + * @return a value which will be returned by the visit method. + */ + T get(); + +} diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/SchemaVisitorAction.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/SchemaVisitorAction.java new file mode 100644 index 00000000000..81157aad0a3 --- /dev/null +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/SchemaVisitorAction.java @@ -0,0 +1,44 @@ + /* + * Copyright (c) 2001 - 2016, Zoltan Farkas All Rights Reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +package org.apache.avro.compiler.schema; + +/** + * @author zoly + */ +public enum SchemaVisitorAction { + + /** + * continue visit. + */ + CONTINUE, + /** + * terminate visit. + */ + TERMINATE, + /** + * when returned from pre non terminal visit method the children of the non terminal are skipped. + * afterVisitNonTerminal for the current schema will not be invoked. + */ + SKIP_SUBTREE, + /** + * Skip visiting the siblings of this schema. + */ + SKIP_SIBLINGS; + +} diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/Schemas.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/Schemas.java new file mode 100644 index 00000000000..82cf0bbe943 --- /dev/null +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/Schemas.java @@ -0,0 +1,245 @@ +package org.apache.avro.compiler.schema; + +import avro.shaded.com.google.common.base.Function; +import avro.shaded.com.google.common.base.Supplier; +import avro.shaded.com.google.common.collect.Lists; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Deque; +import java.util.IdentityHashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import javax.annotation.ParametersAreNonnullByDefault; +import org.apache.avro.JsonProperties; +import org.apache.avro.LogicalType; +import org.apache.avro.Schema; +import org.apache.avro.Schema.Field; +import org.apache.avro.compiler.specific.SpecificCompiler; + +/** + * Avro Schema utilities, to traverse... + * + * @author zoly + */ +@ParametersAreNonnullByDefault +public final class Schemas { + + private Schemas() { + } + + public static void copyAliases(final Schema from, final Schema to) { + switch (from.getType()) { // only named types. + case RECORD: + case ENUM: + case FIXED: + Set aliases = from.getAliases(); + for (String alias : aliases) { + to.addAlias(alias); + } + } + } + + public static void copyAliases(final Schema.Field from, final Schema.Field to) { + Set aliases = from.aliases(); + for (String alias : aliases) { + to.addAlias(alias); + } + } + + public static void copyLogicalTypes(final Schema from, final Schema to) { + LogicalType logicalType = from.getLogicalType(); + if (logicalType != null) { + logicalType.addToSchema(to); + } + } + + public static void copyProperties(final JsonProperties from, final JsonProperties to) { + Map objectProps = from.getObjectProps(); + for (Map.Entry entry : objectProps.entrySet()) { + to.addProp(entry.getKey(), entry.getValue()); + } + } + + public static boolean hasGeneratedJavaClass(final Schema schema) { + Schema.Type type = schema.getType(); + switch (type) { + case ENUM: + case RECORD: + case FIXED: + return true; + default: + return false; + } + } + + public static String getJavaClassName(final Schema schema) { + String namespace = schema.getNamespace(); + if (namespace == null || namespace.isEmpty()) { + return SpecificCompiler.mangle(schema.getName()); + } else { + return namespace + '.' + SpecificCompiler.mangle(schema.getName()); + } + } + + /** + * depth first visit. + * + * @param start + * @param visitor + */ + public static T visit(final Schema start, final SchemaVisitor visitor) { + // Set of Visited Schemas + IdentityHashMap visited = new IdentityHashMap(); + // Stack that contains the Schams to process and afterVisitNonTerminal functions. + // Deque>> + // Using either has a cost which we want to avoid... + Deque dq = new ArrayDeque(); + dq.addLast(start); + Object current; + while ((current = dq.pollLast()) != null) { + if (current instanceof Supplier) { + // we are executing a non terminal post visit. + SchemaVisitorAction action = ((Supplier) current).get(); + switch (action) { + case CONTINUE: + break; + case SKIP_SUBTREE: + throw new UnsupportedOperationException(); + case SKIP_SIBLINGS: + //CHECKSTYLE:OFF InnerAssignment + while ((current = dq.getLast()) instanceof Schema) { + // just skip + } + //CHECKSTYLE:ON + dq.addLast(current); + break; + case TERMINATE: + return visitor.get(); + default: + throw new UnsupportedOperationException("Invalid action " + action); + } + } else { + Schema schema = (Schema) current; + boolean terminate; + if (!visited.containsKey(schema)) { + Schema.Type type = schema.getType(); + switch (type) { + case ARRAY: + terminate = visitNonTerminal(visitor, schema, dq, Arrays.asList(schema.getElementType())); + visited.put(schema, schema); + break; + case RECORD: + terminate = visitNonTerminal(visitor, schema, dq, + Lists.transform(Lists.reverse(schema.getFields()), new Function() { + @Override + public Schema apply(Field f) { + return f.schema(); + } + })); + visited.put(schema, schema); + break; + case UNION: + terminate = visitNonTerminal(visitor, schema, dq, schema.getTypes()); + visited.put(schema, schema); + break; + case MAP: + terminate = visitNonTerminal(visitor, schema, dq, Arrays.asList(schema.getValueType())); + visited.put(schema, schema); + break; + case NULL: + case BOOLEAN: + case BYTES: + case DOUBLE: + case ENUM: + case FIXED: + case FLOAT: + case INT: + case LONG: + case STRING: + terminate = visitTerminal(visitor, schema, dq); + break; + default: + throw new UnsupportedOperationException("Invalid type " + type); + } + + } else { + terminate = visitTerminal(visitor, schema, dq); + } + if (terminate) { + return visitor.get(); + } + } + } + return visitor.get(); + } + + private static boolean visitNonTerminal(final SchemaVisitor visitor, + final Schema schema, final Deque dq, + final Iterable itSupp) { + SchemaVisitorAction action = visitor.visitNonTerminal(schema); + switch (action) { + case CONTINUE: + dq.addLast(new Supplier() { + @Override + public SchemaVisitorAction get() { + return visitor.afterVisitNonTerminal(schema); + } + }); + Iterator it = itSupp.iterator(); + while (it.hasNext()) { + Schema child = it.next(); + dq.addLast(child); + } + break; + case SKIP_SUBTREE: + dq.addLast(new Supplier() { + @Override + public SchemaVisitorAction get() { + return visitor.afterVisitNonTerminal(schema); + } + }); + break; + case SKIP_SIBLINGS: + Object current; + //CHECKSTYLE:OFF InnerAssignment + while ((current = dq.getLast()) instanceof Schema) { + // just skip + } + //CHECKSTYLE:ON + dq.addLast(current); + break; + case TERMINATE: + return true; + default: + throw new UnsupportedOperationException("Invalid action " + action + " for " + schema); + } + return false; + } + + private static boolean visitTerminal(final SchemaVisitor visitor, final Schema schema, + final Deque dq) { + SchemaVisitorAction action = visitor.visitTerminal(schema); + switch (action) { + case CONTINUE: + break; + case SKIP_SUBTREE: + throw new UnsupportedOperationException("Invalid action " + action + " for " + schema); + case SKIP_SIBLINGS: + Object current; + //CHECKSTYLE:OFF InnerAssignment + while ((current = dq.getLast()) instanceof Schema) { + // just skip + } + //CHECKSTYLE:ON + dq.addLast(current); + break; + case TERMINATE: + return true; + default: + throw new UnsupportedOperationException("Invalid action " + action + " for " + schema); + } + return false; + } + +} From 884fbabd3a07e6442dac9a43f34eb9ca0e5a0cbc Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Sun, 2 Apr 2017 11:28:00 -0400 Subject: [PATCH 15/22] [merge] marge with latest changes --- CHANGES.txt | 30 ++ doc/src/content/xdocs/gettingstartedjava.xml | 2 +- lang/c++/CMakeLists.txt | 27 +- lang/c++/FindSnappy.cmake | 54 +++ lang/c++/README | 32 +- lang/c++/api/DataFile.hh | 9 +- lang/c++/api/GenericDatum.hh | 32 +- lang/c++/api/Specific.hh | 24 ++ lang/c++/impl/Compiler.cc | 37 +- lang/c++/impl/DataFile.cc | 116 ++++- lang/c++/impl/Generic.cc | 4 - lang/c++/impl/avrogencpp.cc | 42 +- lang/c++/impl/json/JsonIO.cc | 12 +- lang/c++/impl/json/JsonIO.hh | 8 +- lang/c++/impl/parsing/Symbol.hh | 3 + lang/c++/jsonschemas/bigrecord | 4 + lang/c++/jsonschemas/bigrecord_r | 6 +- lang/c++/jsonschemas/crossref | 28 ++ lang/c++/jsonschemas/padded_record | 14 + lang/c++/jsonschemas/primitivetypes | 15 + lang/c++/jsonschemas/tree1 | 25 ++ lang/c++/jsonschemas/tree2 | 18 + lang/c++/test/AvrogencppTests.cc | 4 + lang/c++/test/CodecTests.cc | 74 +++- lang/c++/test/DataFileTests.cc | 40 +- lang/c++/test/JsonTests.cc | 3 + lang/c/src/codec.c | 3 + .../src/main/java/org/apache/avro/Schema.java | 3 + .../java/org/apache/avro/SchemaBuilder.java | 4 + .../java/org/apache/avro/file/BZip2Codec.java | 2 +- .../org/apache/avro/file/DeflateCodec.java | 2 +- .../java/org/apache/avro/file/NullCodec.java | 2 +- .../org/apache/avro/file/SnappyCodec.java | 2 +- .../java/org/apache/avro/file/XZCodec.java | 2 +- .../avro/specific/SpecificRecordBase.java | 2 +- .../apache/avro/util/WeakIdentityHashMap.java | 6 + .../avro/util/internal/JacksonUtils.java | 3 +- .../test/java/org/apache/avro/TestFixed.java | 21 +- .../compiler/src/test/idl/input/cycle.avdl | 2 +- .../compiler/src/test/idl/output/cycle.avpr | 3 +- .../ipc/TestNettyServerWithCallbacks.java | 35 +- lang/java/pom.xml | 60 ++- lang/java/tools/pom.xml | 5 - .../java/org/apache/avro/tool/ConcatTool.java | 28 +- .../main/java/org/apache/avro/tool/Util.java | 39 +- .../tools/src/main/resources/log4j.properties | 22 + .../org/apache/avro/tool/TestCatTool.java | 14 + .../org/apache/avro/tool/TestConcatTool.java | 79 ++++ lang/ruby/Manifest | 2 + lang/ruby/lib/avro.rb | 1 + lang/ruby/lib/avro/schema.rb | 39 +- lang/ruby/lib/avro/schema_validator.rb | 196 +++++++++ lang/ruby/test/test_schema.rb | 56 +-- lang/ruby/test/test_schema_validator.rb | 402 ++++++++++++++++++ 54 files changed, 1439 insertions(+), 259 deletions(-) create mode 100644 lang/c++/FindSnappy.cmake create mode 100644 lang/c++/jsonschemas/crossref create mode 100644 lang/c++/jsonschemas/padded_record create mode 100644 lang/c++/jsonschemas/primitivetypes create mode 100644 lang/c++/jsonschemas/tree1 create mode 100644 lang/c++/jsonschemas/tree2 create mode 100644 lang/java/tools/src/main/resources/log4j.properties create mode 100644 lang/ruby/lib/avro/schema_validator.rb create mode 100644 lang/ruby/test/test_schema_validator.rb diff --git a/CHANGES.txt b/CHANGES.txt index fbb1c2f4fea..bd032046b6c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -40,6 +40,14 @@ Trunk (not yet released) AVRO-1897: Fix build issues due to VERSION.txt newline, avro-tools. (Suraj Acharya via blue) + AVRO-1993: C++ Byte ordering macro does not work on FreeBSD (thiru) + + AVRO-1975: Upgrade java dependencies (gabor) + + AVRO-1960: Add log4j properties for avro-tools + + AVRO-1748. Add Snappy Compression to C++ DataFile (J. Langley via thiru) + BUG FIXES AVRO-1741: Python3: Fix error when codec is not in the header. @@ -117,6 +125,28 @@ Trunk (not yet released) AVRO-1954: Java: Schema.Field.defaultVal() generates: Unknown datum type (Nandor Kollar via tomwhite) + AVRO-1930: JsonParser doesn't handle integer scientific notation (Pietro Cerutti via thiru) + + AVRO-1912: C++ Resolving Decoding doesn't work if element removed from record in array. (via thiru) + + AVRO-1866. JsonNullFormatter fwd-declared as class, defined as struct ( Pietro Cerutti via thiru) + + AVRO-1750. GenericDatum API behavior breaking change (thiru) + + AVRO-1995: JSON Parser does not properly check current state (Victor Mota via thiru) + + AVRO-1216. Setting precision for the output stream (John McClean via thiru) + + AVRO-1813: Incorrect link to build instructions in Java Getting Started (Pietro Menna via gabor) + + AVRO-1937: C++ generator for recursive structure crashes (thiru) + + AVRO-1892. C++ library cannot parse unions with default values (Hua Zhang via thiru) + + AVRO-1994. C++ Code Generator Generates Invalid Code if Field is of type Null (Darryl Green via thiru) + + AVRO-1997. Avro Field.defaultVal broken for Fixed fields. (Zoltan Farkasi via thiru) + Avro 1.8.1 (14 May 2016) INCOMPATIBLE CHANGES diff --git a/doc/src/content/xdocs/gettingstartedjava.xml b/doc/src/content/xdocs/gettingstartedjava.xml index ea760b381a5..6474ebc9219 100644 --- a/doc/src/content/xdocs/gettingstartedjava.xml +++ b/doc/src/content/xdocs/gettingstartedjava.xml @@ -93,7 +93,7 @@

You may also build the required Avro jars from source. Building Avro is beyond the scope of this guide; see the Build + href="https://cwiki.apache.org/AVRO/Build+Documentation">Build Documentation page in the wiki for more information.

diff --git a/lang/c++/CMakeLists.txt b/lang/c++/CMakeLists.txt index e8efe86dd17..be392152128 100644 --- a/lang/c++/CMakeLists.txt +++ b/lang/c++/CMakeLists.txt @@ -36,6 +36,7 @@ set (AVRO_VERSION_MAJOR ${AVRO_VERSION}) set (AVRO_VERSION_MINOR "0") project (Avro-cpp) +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR}) if (WIN32 AND NOT CYGWIN AND NOT MSYS) add_definitions (/EHa) @@ -61,6 +62,17 @@ endif () find_package (Boost 1.38 REQUIRED COMPONENTS filesystem system program_options iostreams) +find_package(Snappy) +if (SNAPPY_FOUND) + set(SNAPPY_PKG libsnappy) + add_definitions(-DSNAPPY_CODEC_AVAILABLE) + message("Enabled snappy codec") +else (SNAPPY_FOUND) + set(SNAPPY_PKG "") + set(SNAPPY_LIBRARIES "") + message("Disabled snappy codec. libsnappy not found.") +endif (SNAPPY_FOUND) + add_definitions (${Boost_LIB_DIAGNOSTIC_DEFINITIONS}) include_directories (api ${CMAKE_CURRENT_BINARY_DIR} ${Boost_INCLUDE_DIRS}) @@ -98,11 +110,11 @@ set_target_properties (avrocpp PROPERTIES set_target_properties (avrocpp_s PROPERTIES VERSION ${AVRO_VERSION_MAJOR}.${AVRO_VERSION_MINOR}) -target_link_libraries (avrocpp ${Boost_LIBRARIES}) +target_link_libraries (avrocpp ${Boost_LIBRARIES} ${SNAPPY_LIBRARIES}) add_executable (precompile test/precompile.cc) -target_link_libraries (precompile avrocpp_s ${Boost_LIBRARIES}) +target_link_libraries (precompile avrocpp_s ${Boost_LIBRARIES} ${SNAPPY_LIBRARIES}) macro (gen file ns) add_custom_command (OUTPUT ${file}.hh @@ -125,15 +137,19 @@ gen (union_conflict uc) gen (recursive rec) gen (reuse ru) gen (circulardep cd) +gen (tree1 tr1) +gen (tree2 tr2) +gen (crossref cr) +gen (primitivetypes pt) add_executable (avrogencpp impl/avrogencpp.cc) -target_link_libraries (avrogencpp avrocpp_s ${Boost_LIBRARIES}) +target_link_libraries (avrogencpp avrocpp_s ${Boost_LIBRARIES} ${SNAPPY_LIBRARIES}) enable_testing() macro (unittest name) add_executable (${name} test/${name}.cc) - target_link_libraries (${name} avrocpp ${Boost_LIBRARIES}) + target_link_libraries (${name} avrocpp ${Boost_LIBRARIES} ${SNAPPY_LIBRARIES}) add_test (NAME ${name} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${name}) endmacro (unittest) @@ -153,7 +169,8 @@ unittest (CompilerTests) add_dependencies (AvrogencppTests bigrecord_hh bigrecord_r_hh bigrecord2_hh tweet_hh union_array_union_hh union_map_union_hh union_conflict_hh - recursive_hh reuse_hh circulardep_hh empty_record_hh) + recursive_hh reuse_hh circulardep_hh tree1_hh tree2_hh crossref_hh + primitivetypes_hh empty_record_hh) include (InstallRequiredSystemLibraries) diff --git a/lang/c++/FindSnappy.cmake b/lang/c++/FindSnappy.cmake new file mode 100644 index 00000000000..e9053afc4f7 --- /dev/null +++ b/lang/c++/FindSnappy.cmake @@ -0,0 +1,54 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Tries to find Snappy headers and libraries. +# +# Usage of this module as follows: +# +# find_package(Snappy) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# SNAPPY_ROOT_DIR Set this variable to the root installation of +# Snappy if the module has problems finding +# the proper installation path. +# +# Variables defined by this module: +# +# SNAPPY_FOUND System has Snappy libs/headers +# SNAPPY_LIBRARIES The Snappy libraries +# SNAPPY_INCLUDE_DIR The location of Snappy headers + +find_path(SNAPPY_INCLUDE_DIR + NAMES snappy.h + HINTS ${SNAPPY_ROOT_DIR}/include) + +find_library(SNAPPY_LIBRARIES + NAMES snappy + HINTS ${SNAPPY_ROOT_DIR}/lib) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Snappy DEFAULT_MSG + SNAPPY_LIBRARIES + SNAPPY_INCLUDE_DIR) + +mark_as_advanced( + SNAPPY_ROOT_DIR + SNAPPY_LIBRARIES + SNAPPY_INCLUDE_DIR) diff --git a/lang/c++/README b/lang/c++/README index 7a799711afe..f1e9b66292c 100644 --- a/lang/c++/README +++ b/lang/c++/README @@ -23,18 +23,21 @@ The C++ port is thus far incomplete. Currently, it contains: objects of the same data types, and the code to serialize and parse it. -What's missing: Defaults are not yet supported. And the file and rpc -containers are not yet implemented. Documentation, sparse. +What's missing: Rpc containers are not yet implemented. Documentation is sparse. INSTRUCTIONS -To compile requires boost headers, and the boost regex library. -Additionally, to generate the avro spec compiler requires flex and bison. +Pre-requisites: + +To compile requires boost headers, and the boost regex library. Optionally, it requires Snappy compression library. If Snappy is available, it builds support for Snappy compression and skips it otherwise. (Please see your OS-specific instructions on how to install Boost and Snappy for your OS). + To build one requires cmake 2.6 or later. -To generate a Makefile under Unix or Cygwin use: +To generate a Makefile under Unix, MacOS (using GNU) or Cygwin use: -cmake -G "Unix Makefiles" +mkdir build +cd build +cmake -G "Unix Makefiles" .. If it doesn't work, either you are missing some packages (boost, flex or bison), or you need to help configure locate them. @@ -42,11 +45,7 @@ or you need to help configure locate them. If the Makefile is configured correctly, then you can make and run tests: make - ./build/unittest - ./build/buffertest - ./build/testgentest - ./build/CodecTests - ./build/StreamTests + ctest To install @@ -54,6 +53,17 @@ To install and then untar the generated .tar.gz file. +To build and test on MacOS (using Xcode) + +mkdir build.mac +cd build.mac +cmake -G Xcode + +xcodebuild -configuration Release +ctest -C Release + +If debug version is required, replace 'Release' above with 'Debug'. + Note: The LICENSE and NOTICE files in the lang/c++ source directory are used to build the binary distribution. The LICENSE and NOTICE information for the Avro C++ source distribution is in the root directory. diff --git a/lang/c++/api/DataFile.hh b/lang/c++/api/DataFile.hh index 98779b6b101..bff30977030 100644 --- a/lang/c++/api/DataFile.hh +++ b/lang/c++/api/DataFile.hh @@ -40,7 +40,12 @@ namespace avro { /** Specify type of compression to use when writing data files. */ enum Codec { NULL_CODEC, - DEFLATE_CODEC + DEFLATE_CODEC, + +#ifdef SNAPPY_CODEC_AVAILABLE + SNAPPY_CODEC +#endif + }; /** @@ -185,7 +190,7 @@ class AVRO_DECL DataFileReaderBase : boost::noncopyable { // for compressed buffer boost::scoped_ptr os_; std::vector compressed_; - + std::string uncompressed; void readHeader(); bool readDataBlock(); diff --git a/lang/c++/api/GenericDatum.hh b/lang/c++/api/GenericDatum.hh index 5efcb7f4b81..2b1b3a42455 100644 --- a/lang/c++/api/GenericDatum.hh +++ b/lang/c++/api/GenericDatum.hh @@ -66,18 +66,14 @@ public: /** * The avro data type this datum holds. */ - Type type() const { - return type_; - } + Type type() const; /** * Returns the value held by this datum. * T The type for the value. This must correspond to the * avro type returned by type(). */ - template const T& value() const { - return *boost::any_cast(&value_); - } + template const T& value() const; /** * Returns the reference to the value held by this datum, which @@ -88,9 +84,7 @@ public: * T The type for the value. This must correspond to the * avro type returned by type(). */ - template T& value() { - return *boost::any_cast(&value_); - } + template T& value(); /** * Returns true if and only if this datum is a union. @@ -153,7 +147,7 @@ public: GenericDatum(const NodePtr& schema, const T& v) : type_(schema->type()) { init(schema); - value() = v; + *boost::any_cast(&value_) = v; } /** @@ -493,6 +487,24 @@ public: } }; +inline Type GenericDatum::type() const { + return (type_ == AVRO_UNION) ? + boost::any_cast(&value_)->datum().type() : + type_; +} + +template T& GenericDatum::value() { + return (type_ == AVRO_UNION) ? + boost::any_cast(&value_)->datum().value() : + *boost::any_cast(&value_); +} + +template const T& GenericDatum::value() const { + return (type_ == AVRO_UNION) ? + boost::any_cast(&value_)->datum().value() : + *boost::any_cast(&value_); +} + inline size_t GenericDatum::unionBranch() const { return boost::any_cast(&value_)->currentBranch(); } diff --git a/lang/c++/api/Specific.hh b/lang/c++/api/Specific.hh index ef50318a548..0a00fb3ccdf 100644 --- a/lang/c++/api/Specific.hh +++ b/lang/c++/api/Specific.hh @@ -25,6 +25,7 @@ #include #include "boost/array.hpp" +#include "boost/blank.hpp" #include "Config.hh" #include "Encoder.hh" @@ -46,6 +47,8 @@ */ namespace avro { +typedef boost::blank null; + template void encode(Encoder& e, const T& t); template void decode(Decoder& d, T& t); @@ -289,6 +292,27 @@ template struct codec_traits > { } }; +/** +* codec_traits for Avro null. +*/ +template <> struct codec_traits { + /** + * Encodes a given value. + */ + static void encode(Encoder& e, const avro::null&) { + e.encodeNull(); + } + + /** + * Decodes into a given value. + */ + static void decode(Decoder& d, avro::null&) { + d.decodeNull(); + } +}; + + + /** * Generic encoder function that makes use of the codec_traits. */ diff --git a/lang/c++/impl/Compiler.cc b/lang/c++/impl/Compiler.cc index 96708449df7..be5fe3f862b 100644 --- a/lang/c++/impl/Compiler.cc +++ b/lang/c++/impl/Compiler.cc @@ -79,7 +79,7 @@ static bool isFullName(const string& s) { return s.find('.') != string::npos; } - + static Name getName(const string& name, const string& ns) { return (isFullName(name)) ? Name(name) : Name(name, ns); @@ -143,7 +143,7 @@ const int64_t getLongField(const Entity& e, const Object& m, ensureType(it->second, fieldName); return it->second.longValue(); } - + struct Field { const string& name; const NodePtr schema; @@ -283,32 +283,9 @@ static GenericDatum makeGenericDatum(NodePtr n, const Entity& e, case AVRO_UNION: { GenericUnion result(n); - string name; - Entity e2; - if (e.type() == json::etNull) { - name = "null"; - e2 = e; - } else { - assertType(e, json::etObject); - const map& v = e.objectValue(); - if (v.size() != 1) { - throw Exception(boost::format("Default value for " - "union has more than one field: %1%") % e.toString()); - } - map::const_iterator it = v.begin(); - name = it->first; - e2 = it->second; - } - for (size_t i = 0; i < n->leaves(); ++i) { - const NodePtr& b = n->leafAt(i); - if (nameof(b) == name) { - result.selectBranch(i); - result.datum() = makeGenericDatum(b, e2, st); - return GenericDatum(n, result); - } - } - throw Exception(boost::format("Invalid default value %1%") % - e.toString()); + result.selectBranch(0); + result.datum() = makeGenericDatum(n->leafAt(0), e, st); + return GenericDatum(n, result); } case AVRO_FIXED: assertType(e, json::etString); @@ -334,12 +311,12 @@ static Field makeField(const Entity& e, SymbolTable& st, const string& ns) static NodePtr makeRecordNode(const Entity& e, const Name& name, const Object& m, SymbolTable& st, const string& ns) -{ +{ const Array& v = getArrayField(e, m, "fields"); concepts::MultiAttribute fieldNames; concepts::MultiAttribute fieldValues; vector defaultValues; - + for (Array::const_iterator it = v.begin(); it != v.end(); ++it) { Field f = makeField(*it, st, ns); fieldNames.add(f.name); diff --git a/lang/c++/impl/DataFile.cc b/lang/c++/impl/DataFile.cc index 035dd27404a..ee8f62c6a4b 100644 --- a/lang/c++/impl/DataFile.cc +++ b/lang/c++/impl/DataFile.cc @@ -26,6 +26,11 @@ #include #include #include +#include // for boost::crc_32_type + +#ifdef SNAPPY_CODEC_AVAILABLE +#include +#endif namespace avro { using std::auto_ptr; @@ -43,6 +48,10 @@ const string AVRO_CODEC_KEY("avro.codec"); const string AVRO_NULL_CODEC("null"); const string AVRO_DEFLATE_CODEC("deflate"); +#ifdef SNAPPY_CODEC_AVAILABLE +const string AVRO_SNAPPY_CODEC = "snappy"; +#endif + const size_t minSyncInterval = 32; const size_t maxSyncInterval = 1u << 30; const size_t defaultSyncInterval = 64 * 1024; @@ -83,8 +92,12 @@ DataFileWriterBase::DataFileWriterBase(const char* filename, setMetadata(AVRO_CODEC_KEY, AVRO_NULL_CODEC); } else if (codec_ == DEFLATE_CODEC) { setMetadata(AVRO_CODEC_KEY, AVRO_DEFLATE_CODEC); +#ifdef SNAPPY_CODEC_AVAILABLE + } else if (codec_ == SNAPPY_CODEC) { + setMetadata(AVRO_CODEC_KEY, AVRO_SNAPPY_CODEC); +#endif } else { - throw Exception("Unknown codec codec"); + throw Exception(boost::format("Unknown codec: %1%") % codec); } setMetadata(AVRO_SCHEMA_KEY, toString(schema)); @@ -117,13 +130,11 @@ void DataFileWriterBase::sync() encoderPtr_->flush(); std::auto_ptr in = memoryInputStream(*buffer_); copy(*in, *stream_); - } else { + } else if (codec_ == DEFLATE_CODEC) { std::vector buf; { boost::iostreams::filtering_ostream os; - if (codec_ == DEFLATE_CODEC) { - os.push(boost::iostreams::zlib_compressor(get_zlib_params())); - } + os.push(boost::iostreams::zlib_compressor(get_zlib_params())); os.push(boost::iostreams::back_inserter(buf)); const uint8_t* data; size_t len; @@ -139,6 +150,49 @@ void DataFileWriterBase::sync() avro::encode(*encoderPtr_, byteCount); encoderPtr_->flush(); copy(*in, *stream_); +#ifdef SNAPPY_CODEC_AVAILABLE + } else if (codec_ == SNAPPY_CODEC) { + std::vector temp; + std::string compressed; + boost::crc_32_type crc; + { + boost::iostreams::filtering_ostream os; + os.push(boost::iostreams::back_inserter(temp)); + const uint8_t* data; + size_t len; + + std::auto_ptr input = memoryInputStream(*buffer_); + while (input->next(&data, &len)) { + boost::iostreams::write(os, reinterpret_cast(data), + len); + } + } // make sure all is flushed + + crc.process_bytes(reinterpret_cast(&temp[0]), temp.size()); + // For Snappy, add the CRC32 checksum + int32_t checksum = crc(); + + // Now compress + size_t compressed_size = snappy::Compress( + reinterpret_cast(&temp[0]), temp.size(), + &compressed); + temp.clear(); + { + boost::iostreams::filtering_ostream os; + os.push(boost::iostreams::back_inserter(temp)); + boost::iostreams::write(os, compressed.c_str(), compressed_size); + } + temp.push_back((checksum >> 24) & 0xFF); + temp.push_back((checksum >> 16) & 0xFF); + temp.push_back((checksum >> 8) & 0xFF); + temp.push_back(checksum & 0xFF); + std::auto_ptr in = memoryInputStream( + reinterpret_cast(&temp[0]), temp.size()); + int64_t byteCount = temp.size(); + avro::encode(*encoderPtr_, byteCount); + encoderPtr_->flush(); + copy(*in, *stream_); +#endif } encoderPtr_->init(*stream_); @@ -320,7 +374,7 @@ bool DataFileReaderBase::readDataBlock() if (codec_ == NULL_CODEC) { dataDecoder_->init(*st); dataStream_ = st; - } else { + } else if (codec_ == DEFLATE_CODEC) { compressed_.clear(); const uint8_t* data; size_t len; @@ -329,17 +383,52 @@ bool DataFileReaderBase::readDataBlock() } // boost::iostreams::write(os, reinterpret_cast(data), len); os_.reset(new boost::iostreams::filtering_istream()); - if (codec_ == DEFLATE_CODEC) { - os_->push(boost::iostreams::zlib_decompressor(get_zlib_params())); - } else { - throw Exception("Bad codec"); - } + os_->push(boost::iostreams::zlib_decompressor(get_zlib_params())); os_->push(boost::iostreams::basic_array_source( &compressed_[0], compressed_.size())); std::auto_ptr in = istreamInputStream(*os_); dataDecoder_->init(*in); dataStream_ = in; +#ifdef SNAPPY_CODEC_AVAILABLE + } else if (codec_ == SNAPPY_CODEC) { + boost::crc_32_type crc; + uint32_t checksum = 0; + compressed_.clear(); + uncompressed.clear(); + const uint8_t* data; + size_t len; + while (st->next(&data, &len)) { + compressed_.insert(compressed_.end(), data, data + len); + } + len = compressed_.size(); + int b1 = compressed_[len - 4] & 0xFF; + int b2 = compressed_[len - 3] & 0xFF; + int b3 = compressed_[len - 2] & 0xFF; + int b4 = compressed_[len - 1] & 0xFF; + + checksum = (b1 << 24) + (b2 << 16) + (b3 << 8) + (b4); + if (!snappy::Uncompress(reinterpret_cast(&compressed_[0]), + len - 4, &uncompressed)) { + throw Exception( + "Snappy Compression reported an error when decompressing"); + } + crc.process_bytes(uncompressed.c_str(), uncompressed.size()); + uint32_t c = crc(); + if (checksum != c) { + throw Exception(boost::format("Checksum did not match for Snappy compression: Expected: %1%, computed: %2%") % checksum % c); + } + os_.reset(new boost::iostreams::filtering_istream()); + os_->push( + boost::iostreams::basic_array_source(uncompressed.c_str(), + uncompressed.size())); + std::auto_ptr in = istreamInputStream(*os_); + + dataDecoder_->init(*in); + dataStream_ = in; +#endif + } else { + throw Exception("Bad codec"); } return true; } @@ -387,6 +476,11 @@ void DataFileReaderBase::readHeader() it = metadata_.find(AVRO_CODEC_KEY); if (it != metadata_.end() && toString(it->second) == AVRO_DEFLATE_CODEC) { codec_ = DEFLATE_CODEC; +#ifdef SNAPPY_CODEC_AVAILABLE + } else if (it != metadata_.end() + && toString(it->second) == AVRO_SNAPPY_CODEC) { + codec_ = SNAPPY_CODEC; +#endif } else { codec_ = NULL_CODEC; if (it != metadata_.end() && toString(it->second) != AVRO_NULL_CODEC) { diff --git a/lang/c++/impl/Generic.cc b/lang/c++/impl/Generic.cc index 884fadb31ca..9eaa56f80a4 100644 --- a/lang/c++/impl/Generic.cc +++ b/lang/c++/impl/Generic.cc @@ -58,8 +58,6 @@ void GenericReader::read(GenericDatum& datum, Decoder& d, bool isResolving) { if (datum.isUnion()) { datum.selectBranch(d.decodeUnionIndex()); - read(datum.value().datum(), d, isResolving); - return; } switch (datum.type()) { case AVRO_NULL: @@ -176,8 +174,6 @@ void GenericWriter::write(const GenericDatum& datum, Encoder& e) { if (datum.isUnion()) { e.encodeUnionIndex(datum.unionBranch()); - write(datum.value().datum(), e); - return; } switch (datum.type()) { case AVRO_NULL: diff --git a/lang/c++/impl/avrogencpp.cc b/lang/c++/impl/avrogencpp.cc index a6a858d546c..a44fe7dfe39 100644 --- a/lang/c++/impl/avrogencpp.cc +++ b/lang/c++/impl/avrogencpp.cc @@ -173,6 +173,8 @@ string CodeGen::cppTypeOf(const NodePtr& n) return cppTypeOf(resolveSymbol(n)); case avro::AVRO_UNION: return fullname(done[n]); + case avro::AVRO_NULL: + return "avro::null"; default: return "$Undefined$"; } @@ -215,6 +217,7 @@ static string cppNameOf(const NodePtr& n) string CodeGen::generateRecordType(const NodePtr& n) { size_t c = n->leaves(); + string decoratedName = decorate(n->name()); vector types; for (size_t i = 0; i < c; ++i) { types.push_back(generateType(n->leafAt(i))); @@ -225,7 +228,6 @@ string CodeGen::generateRecordType(const NodePtr& n) return it->second; } - string decoratedName = decorate(n->name()); os_ << "struct " << decoratedName << " {\n"; if (! noUnion_) { for (size_t i = 0; i < c; ++i) { @@ -264,7 +266,7 @@ string CodeGen::generateRecordType(const NodePtr& n) } os_ << " { }\n"; os_ << "};\n\n"; - return decorate(n->name()); + return decoratedName; } void makeCanonical(string& s, bool foldCase) @@ -391,7 +393,7 @@ string CodeGen::generateUnionType(const NodePtr& n) pendingConstructors.push_back(PendingConstructor(result, types[0], n->leafAt(0)->type() != avro::AVRO_NULL)); os_ << "};\n\n"; - + return result; } @@ -425,9 +427,31 @@ string CodeGen::doGenerateType(const NodePtr& n) case avro::AVRO_FIXED: return cppTypeOf(n); case avro::AVRO_ARRAY: - return "std::vector<" + generateType(n->leafAt(0)) + " >"; + { + const NodePtr& ln = n->leafAt(0); + string dn; + if (doing.find(n) == doing.end()) { + doing.insert(n); + dn = generateType(ln); + doing.erase(n); + } else { + dn = generateDeclaration(ln); + } + return "std::vector<" + dn + " >"; + } case avro::AVRO_MAP: - return "std::mapleafAt(1)) + " >"; + { + const NodePtr& ln = n->leafAt(1); + string dn; + if (doing.find(n) == doing.end()) { + doing.insert(n); + dn = generateType(ln); + doing.erase(n); + } else { + dn = generateDeclaration(ln); + } + return "std::map"; + } case avro::AVRO_RECORD: return generateRecordType(n); case avro::AVRO_ENUM: @@ -478,7 +502,7 @@ void CodeGen::generateEnumTraits(const NodePtr& n) string dname = decorate(n->name()); string fn = fullname(dname); size_t c = n->names(); - string first; + string first; string last; if (!ns_.empty()) { @@ -495,7 +519,7 @@ void CodeGen::generateEnumTraits(const NodePtr& n) } os_ << "template<> struct codec_traits<" << fn << "> {\n" << " static void encode(Encoder& e, " << fn << " v) {\n" - << " if (v < " << first << " || v > " << last << ")\n" + << " if (v < " << first << " || v > " << last << ")\n" << " {\n" << " std::ostringstream error;\n" << " error << \"enum value \" << v << \" is out of bound for " << fn << " and cannot be encoded\";\n" @@ -505,7 +529,7 @@ void CodeGen::generateEnumTraits(const NodePtr& n) << " }\n" << " static void decode(Decoder& d, " << fn << "& v) {\n" << " size_t index = d.decodeEnum();\n" - << " if (index < " << first << " || index > " << last << ")\n" + << " if (index < " << first << " || index > " << last << ")\n" << " {\n" << " std::ostringstream error;\n" << " error << \"enum value \" << index << \" is out of bound for " << fn << " and cannot be decoded\";\n" @@ -650,7 +674,7 @@ void CodeGen::generateTraits(const NodePtr& n) void CodeGen::emitCopyright() { - os_ << + os_ << "/**\n" " * Licensed to the Apache Software Foundation (ASF) under one\n" " * or more contributor license agreements. See the NOTICE file\n" diff --git a/lang/c++/impl/json/JsonIO.cc b/lang/c++/impl/json/JsonIO.cc index 2e7d82fc4db..be5cc2f8bcd 100644 --- a/lang/c++/impl/json/JsonIO.cc +++ b/lang/c++/impl/json/JsonIO.cc @@ -76,7 +76,7 @@ JsonParser::Token JsonParser::doAdvance() { char ch = next(); if (ch == ']') { - if (curState == stArray0 || stArrayN) { + if (curState == stArray0 || curState == stArrayN) { curState = stateStack.top(); stateStack.pop(); return tkArrayEnd; @@ -84,7 +84,7 @@ JsonParser::Token JsonParser::doAdvance() throw unexpected(ch); } } else if (ch == '}') { - if (curState == stObject0 || stObjectN) { + if (curState == stObject0 || curState == stObjectN) { curState = stateStack.top(); stateStack.pop(); return tkObjectEnd; @@ -171,6 +171,10 @@ JsonParser::Token JsonParser::tryNumber(char ch) state = 3; sv.push_back(ch); continue; + } else if (ch == 'e' || ch == 'E') { + sv.push_back(ch); + state = 5; + continue; } hasNext = true; } @@ -185,6 +189,10 @@ JsonParser::Token JsonParser::tryNumber(char ch) state = 3; sv.push_back(ch); continue; + } else if (ch == 'e' || ch == 'E') { + sv.push_back(ch); + state = 5; + continue; } hasNext = true; } diff --git a/lang/c++/impl/json/JsonIO.hh b/lang/c++/impl/json/JsonIO.hh index a5ada2db8ba..c87f73a265c 100644 --- a/lang/c++/impl/json/JsonIO.hh +++ b/lang/c++/impl/json/JsonIO.hh @@ -24,6 +24,7 @@ #include #include #include +#include #include #include "Config.hh" @@ -133,7 +134,8 @@ public: } }; -struct AVRO_DECL JsonNullFormatter { +class AVRO_DECL JsonNullFormatter { +public: JsonNullFormatter(StreamWriter&) { } void handleObjectStart() {} @@ -304,7 +306,7 @@ public: void encodeNumber(T t) { sep(); std::ostringstream oss; - oss << t; + oss << boost::lexical_cast(t); const std::string& s = oss.str(); out_.writeBytes(reinterpret_cast(&s[0]), s.size()); sep2(); @@ -314,7 +316,7 @@ public: sep(); std::ostringstream oss; if (boost::math::isfinite(t)) { - oss << t; + oss << boost::lexical_cast(t); } else if (boost::math::isnan(t)) { oss << "NaN"; } else if (t == std::numeric_limits::infinity()) { diff --git a/lang/c++/impl/parsing/Symbol.hh b/lang/c++/impl/parsing/Symbol.hh index a7c09975acf..291175291d9 100644 --- a/lang/c++/impl/parsing/Symbol.hh +++ b/lang/c++/impl/parsing/Symbol.hh @@ -737,6 +737,9 @@ public: if (s.isImplicitAction()) { handler_.handle(s); parsingStack.pop(); + } else if (s.kind() == Symbol::sSkipStart) { + parsingStack.pop(); + skip(*decoder_); } else { break; } diff --git a/lang/c++/jsonschemas/bigrecord b/lang/c++/jsonschemas/bigrecord index 02dbccb2842..ba430a035ed 100644 --- a/lang/c++/jsonschemas/bigrecord +++ b/lang/c++/jsonschemas/bigrecord @@ -102,6 +102,10 @@ { "name": "bytes", "type": "bytes" + }, + { + "name": "null", + "type": "null" } ] } diff --git a/lang/c++/jsonschemas/bigrecord_r b/lang/c++/jsonschemas/bigrecord_r index f079162d374..7c477cd0fb8 100644 --- a/lang/c++/jsonschemas/bigrecord_r +++ b/lang/c++/jsonschemas/bigrecord_r @@ -70,13 +70,11 @@ { "name": "union1WithDefaultValue", "type": [ "string", "int" ], - "default": { - "string": "sval" - } + "default": "sval" }, { "name": "union2WithDefaultValue", - "type": [ "string", "null" ], + "type": [ "null", "string" ], "default": null }, { diff --git a/lang/c++/jsonschemas/crossref b/lang/c++/jsonschemas/crossref new file mode 100644 index 00000000000..fd1d38afcb7 --- /dev/null +++ b/lang/c++/jsonschemas/crossref @@ -0,0 +1,28 @@ +{ + "name": "A", + "type": "record", + "fields": [ + { + "name": "edges", + "type": { + "type": "array", + "items": { + "type": "record", + "name": "B", + "fields": [ + { "name": "child", + "type": [{ + "type": "record", + "name": "C", + "fields": [ + { "name": "x", "type": { "type": "map", "values": "A" } } + ] + }, "int"] + } + ] + } + } + } + ] +} + diff --git a/lang/c++/jsonschemas/padded_record b/lang/c++/jsonschemas/padded_record new file mode 100644 index 00000000000..cac0e971450 --- /dev/null +++ b/lang/c++/jsonschemas/padded_record @@ -0,0 +1,14 @@ +{ + "type" : "record", + "name" : "PaddedRecord", + "fields" : [ + { + "type" : "int", + "name" : "index" + }, + { + "type" : "bytes", + "name" : "padding" + } + ] +} diff --git a/lang/c++/jsonschemas/primitivetypes b/lang/c++/jsonschemas/primitivetypes new file mode 100644 index 00000000000..0512323a329 --- /dev/null +++ b/lang/c++/jsonschemas/primitivetypes @@ -0,0 +1,15 @@ +{ + "name": "TestPrimitiveTypes", + "type": "record", + "fields": [ + { "name": "Null", "type": "null" }, + { "name": "Boolean", "type": "boolean" }, + { "name": "Int", "type": "int" }, + { "name": "Long", "type": "long" }, + { "name": "Float", "type": "float" }, + { "name": "Double", "type": "double" }, + { "name": "Bytes", "type": "bytes" }, + { "name": "String", "type": "string" }, + { "name": "SecondNull", "type": "null" } + ] +} diff --git a/lang/c++/jsonschemas/tree1 b/lang/c++/jsonschemas/tree1 new file mode 100644 index 00000000000..3add01df6c7 --- /dev/null +++ b/lang/c++/jsonschemas/tree1 @@ -0,0 +1,25 @@ +{ + "name": "Node", + "type": "record", + "fields": [ + { + "name": "payload", + "type": "int", + "default": 0 + }, + { + "name": "edges", + "type": { + "type": "array", + "items": { + "type": "record", + "name": "Edge", + "fields": [ + { "name": "child", "type": "Node" }, + { "name": "label", "type": "string" } + ] + } + } + } + ] +} diff --git a/lang/c++/jsonschemas/tree2 b/lang/c++/jsonschemas/tree2 new file mode 100644 index 00000000000..1cb69b40a1a --- /dev/null +++ b/lang/c++/jsonschemas/tree2 @@ -0,0 +1,18 @@ +{ + "name": "Node", + "type": "record", + "fields": [ + { + "name": "payload", + "type": "int", + "default": 0 + }, + { + "name": "edges", + "type": { + "type": "map", + "values": "Node" + } + } + ] +} diff --git a/lang/c++/test/AvrogencppTests.cc b/lang/c++/test/AvrogencppTests.cc index 26d015599f6..1b429433c4a 100644 --- a/lang/c++/test/AvrogencppTests.cc +++ b/lang/c++/test/AvrogencppTests.cc @@ -27,6 +27,10 @@ #include "recursive.hh" #include "circulardep.hh" #include "reuse.hh" +#include "tree1.hh" +#include "tree2.hh" +#include "crossref.hh" +#include "primitivetypes.hh" #include "Compiler.hh" #include diff --git a/lang/c++/test/CodecTests.cc b/lang/c++/test/CodecTests.cc index c0ca1e05b72..f8bbe84d0a3 100644 --- a/lang/c++/test/CodecTests.cc +++ b/lang/c++/test/CodecTests.cc @@ -1274,6 +1274,21 @@ static const TestData4 data4[] = { "[Rc1sI]", { "100", NULL }, 1 }, + // Record of array of record with deleted field as last field + { "{\"type\":\"record\",\"name\":\"outer\",\"fields\":[" + "{\"name\": \"g1\"," + "\"type\":{\"type\":\"array\",\"items\":{" + "\"name\":\"item\",\"type\":\"record\",\"fields\":[" + "{\"name\":\"f1\", \"type\":\"int\"}," + "{\"name\":\"f2\", \"type\": \"long\", \"default\": 0}]}}}]}", "[c1sIL]", + { "10", "11", NULL }, + "{\"type\":\"record\",\"name\":\"outer\",\"fields\":[" + "{\"name\": \"g1\"," + "\"type\":{\"type\":\"array\",\"items\":{" + "\"name\":\"item\",\"type\":\"record\",\"fields\":[" + "{\"name\":\"f1\", \"type\":\"int\"}]}}}]}", "R[c1sI]", + { "10", NULL }, 2 }, + // Enum resolution { "{\"type\":\"enum\",\"name\":\"e\",\"symbols\":[\"x\",\"y\",\"z\"]}", "e2", @@ -1302,20 +1317,12 @@ static const TestData4 data4[] = { "[c2sU1IsU1I]", { "100", "100", NULL } , "{\"type\":\"array\", \"items\": \"int\"}", "[c2sIsI]", { "100", "100", NULL }, 2 }, - { "{\"type\":\"array\", \"items\":[ \"long\", \"int\"]}", - "[c1sU1Ic1sU1I]", { "100", "100", NULL } , - "{\"type\":\"array\", \"items\": \"int\"}", - "[c1sIc1sI]", { "100", "100", NULL }, 2 }, // Map of unions { "{\"type\":\"map\", \"values\":[ \"long\", \"int\"]}", "{c2sS10U1IsS10U1I}", { "k1", "100", "k2", "100", NULL } , "{\"type\":\"map\", \"values\": \"int\"}", "{c2sS10IsS10I}", { "k1", "100", "k2", "100", NULL }, 2 }, - { "{\"type\":\"map\", \"values\":[ \"long\", \"int\"]}", - "{c1sS10U1Ic1sS10U1I}", { "k1", "100", "k2", "100", NULL } , - "{\"type\":\"map\", \"values\": \"int\"}", - "{c1sS10Ic1sS10I}", { "k1", "100", "k2", "100", NULL }, 2 }, // Union + promotion { "\"int\"", "I", { "100", NULL }, @@ -1339,6 +1346,20 @@ static const TestData4 data4[] = { { "1", "100", "10.75", NULL }, 1 }, }; +static const TestData4 data4BinaryOnly[] = { + // Arrray of unions + { "{\"type\":\"array\", \"items\":[ \"long\", \"int\"]}", + "[c1sU1Ic1sU1I]", { "100", "100", NULL } , + "{\"type\":\"array\", \"items\": \"int\"}", + "[c1sIc1sI]", { "100", "100", NULL }, 2 }, + + // Map of unions + { "{\"type\":\"map\", \"values\":[ \"long\", \"int\"]}", + "{c1sS10U1Ic1sS10U1I}", { "k1", "100", "k2", "100", NULL } , + "{\"type\":\"map\", \"values\": \"int\"}", + "{c1sS10Ic1sS10I}", { "k1", "100", "k2", "100", NULL }, 2 }, +}; + #define COUNTOF(x) sizeof(x) / sizeof(x[0]) #define ENDOF(x) (x) + COUNTOF(x) @@ -1405,6 +1426,21 @@ struct BinaryEncoderResolvingDecoderFactory : public BinaryEncoderFactory { } }; +struct JsonEncoderResolvingDecoderFactory { + static EncoderPtr newEncoder(const ValidSchema& schema) { + return jsonEncoder(schema); + } + + static DecoderPtr newDecoder(const ValidSchema& schema) { + return resolvingDecoder(schema, schema, jsonDecoder(schema)); + } + + static DecoderPtr newDecoder(const ValidSchema& writer, + const ValidSchema& reader) { + return resolvingDecoder(writer, reader, jsonDecoder(writer)); + } +}; + struct ValidatingEncoderResolvingDecoderFactory : public ValidatingEncoderFactory { static DecoderPtr newDecoder(const ValidSchema& schema) { @@ -1426,14 +1462,21 @@ void add_tests(boost::unit_test::test_suite& ts) ADD_TESTS(ts, JsonCodec, testCodec, data); ADD_TESTS(ts, JsonPrettyCodec, testCodec, data); ADD_TESTS(ts, BinaryEncoderResolvingDecoderFactory, testCodec, data); + ADD_TESTS(ts, JsonEncoderResolvingDecoderFactory, testCodec, data); ADD_TESTS(ts, ValidatingCodecFactory, testReaderFail, data2); ADD_TESTS(ts, ValidatingCodecFactory, testWriterFail, data2); ADD_TESTS(ts, BinaryEncoderResolvingDecoderFactory, testCodecResolving, data3); + ADD_TESTS(ts, JsonEncoderResolvingDecoderFactory, + testCodecResolving, data3); ADD_TESTS(ts, BinaryEncoderResolvingDecoderFactory, testCodecResolving2, data4); + ADD_TESTS(ts, JsonEncoderResolvingDecoderFactory, + testCodecResolving2, data4); ADD_TESTS(ts, ValidatingEncoderResolvingDecoderFactory, testCodecResolving2, data4); + ADD_TESTS(ts, BinaryEncoderResolvingDecoderFactory, + testCodecResolving2, data4BinaryOnly); ADD_TESTS(ts, ValidatingCodecFactory, testGeneric, data); ADD_TESTS(ts, ValidatingCodecFactory, testGenericResolving, data3); @@ -1470,9 +1513,13 @@ static void testLimits(const EncoderPtr& e, const DecoderPtr& d) e->encodeDouble(std::numeric_limits::infinity()); e->encodeDouble(-std::numeric_limits::infinity()); e->encodeDouble(std::numeric_limits::quiet_NaN()); + e->encodeDouble(std::numeric_limits::max()); + e->encodeDouble(std::numeric_limits::min()); e->encodeFloat(std::numeric_limits::infinity()); e->encodeFloat(-std::numeric_limits::infinity()); e->encodeFloat(std::numeric_limits::quiet_NaN()); + e->encodeFloat(std::numeric_limits::max()); + e->encodeFloat(std::numeric_limits::min()); e->flush(); } @@ -1484,13 +1531,16 @@ static void testLimits(const EncoderPtr& e, const DecoderPtr& d) BOOST_CHECK_EQUAL(d->decodeDouble(), -std::numeric_limits::infinity()); BOOST_CHECK(boost::math::isnan(d->decodeDouble())); + BOOST_CHECK(d->decodeDouble() == std::numeric_limits::max()); + BOOST_CHECK(d->decodeDouble() == std::numeric_limits::min()); BOOST_CHECK_EQUAL(d->decodeFloat(), std::numeric_limits::infinity()); BOOST_CHECK_EQUAL(d->decodeFloat(), -std::numeric_limits::infinity()); BOOST_CHECK(boost::math::isnan(d->decodeFloat())); + BOOST_CHECK_CLOSE(d->decodeFloat(), std::numeric_limits::max(), 0.00011); + BOOST_CHECK_CLOSE(d->decodeFloat(), std::numeric_limits::min(), 0.00011); } - } static void testLimitsBinaryCodec() @@ -1504,9 +1554,13 @@ static void testLimitsJsonCodec() "{ \"name\": \"d1\", \"type\": \"double\" }," "{ \"name\": \"d2\", \"type\": \"double\" }," "{ \"name\": \"d3\", \"type\": \"double\" }," + "{ \"name\": \"d4\", \"type\": \"double\" }," + "{ \"name\": \"d5\", \"type\": \"double\" }," "{ \"name\": \"f1\", \"type\": \"float\" }," "{ \"name\": \"f2\", \"type\": \"float\" }," - "{ \"name\": \"f3\", \"type\": \"float\" }" + "{ \"name\": \"f3\", \"type\": \"float\" }," + "{ \"name\": \"f4\", \"type\": \"float\" }," + "{ \"name\": \"f5\", \"type\": \"float\" }" "]}"; ValidSchema schema = parsing::makeValidSchema(s); testLimits(jsonEncoder(schema), jsonDecoder(schema)); diff --git a/lang/c++/test/DataFileTests.cc b/lang/c++/test/DataFileTests.cc index 95e80b1bc0d..27a7ce9ca50 100644 --- a/lang/c++/test/DataFileTests.cc +++ b/lang/c++/test/DataFileTests.cc @@ -434,11 +434,44 @@ class DataFileTest { } } +#ifdef SNAPPY_CODEC_AVAILABLE + void testSnappy() { + // Add enough objects to span multiple blocks + const size_t number_of_objects = 1000000; + // first create a large file + ValidSchema dschema = avro::compileJsonSchemaFromString(sch); + { + avro::DataFileWriter writer( + filename, dschema, 16 * 1024, avro::SNAPPY_CODEC); + + for (size_t i = 0; i < number_of_objects; ++i) { + ComplexInteger d; + d.re = i; + d.im = 2 * i; + writer.write(d); + } + } + { + avro::DataFileReader reader(filename, dschema); + sleep(1); + std::vector found; + ComplexInteger record; + while (reader.read(record)) { + found.push_back(record.re); + } + BOOST_CHECK_EQUAL(found.size(), number_of_objects); + for (unsigned int i = 0; i < found.size(); ++i) { + BOOST_CHECK_EQUAL(found[i], i); + } + } + } +#endif + void testSchemaReadWrite() { uint32_t a=42; { avro::DataFileWriter df(filename, writerSchema); - df.write(a); + df.write(a); } { @@ -492,7 +525,10 @@ init_unit_test_suite( int argc, char* argv[] ) shared_ptr t6(new DataFileTest("test6.df", dsch, dblsch)); ts->add(BOOST_CLASS_TEST_CASE(&DataFileTest::testZip, t6)); - + shared_ptr t8(new DataFileTest("test8.df", dsch, dblsch)); +#ifdef SNAPPY_CODEC_AVAILABLE + ts->add(BOOST_CLASS_TEST_CASE(&DataFileTest::testSnappy, t8)); +#endif shared_ptr t7(new DataFileTest("test7.df",fsch,fsch)); ts->add(BOOST_CLASS_TEST_CASE(&DataFileTest::testSchemaReadWrite,t7)); ts->add(BOOST_CLASS_TEST_CASE(&DataFileTest::testCleanup,t7)); diff --git a/lang/c++/test/JsonTests.cc b/lang/c++/test/JsonTests.cc index 823f15f7122..79e6c273040 100644 --- a/lang/c++/test/JsonTests.cc +++ b/lang/c++/test/JsonTests.cc @@ -56,6 +56,9 @@ TestData doubleData[] = { { "1.0", etDouble, 1.0 }, { "4.7e3", etDouble, 4700.0 }, { "-7.2e-4", etDouble, -0.00072 }, + { "1e4", etDouble, 10000 }, + { "-1e-4", etDouble, -0.0001 }, + { "-0e0", etDouble, 0.0 }, }; TestData stringData[] = { diff --git a/lang/c/src/codec.c b/lang/c/src/codec.c index 4a2502b5e64..e0d35be35c7 100644 --- a/lang/c/src/codec.c +++ b/lang/c/src/codec.c @@ -21,6 +21,9 @@ # if defined(__APPLE__) # include # define __bswap_32 OSSwapInt32 +# elif defined(__FreeBSD__) +# include +# define __bswap_32 bswap32 # else # include # endif diff --git a/lang/java/avro/src/main/java/org/apache/avro/Schema.java b/lang/java/avro/src/main/java/org/apache/avro/Schema.java index 2019c1f7d9f..53e5e2ea20d 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/Schema.java +++ b/lang/java/avro/src/main/java/org/apache/avro/Schema.java @@ -464,6 +464,8 @@ public boolean equals(Object other) { private boolean defaultValueEquals(JsonNode thatDefaultValue) { if (defaultValue == null) return thatDefaultValue == null; + if (thatDefaultValue == null) + return false; if (Double.isNaN(defaultValue.getDoubleValue())) return Double.isNaN(thatDefaultValue.getDoubleValue()); return defaultValue.equals(thatDefaultValue); @@ -587,6 +589,7 @@ private static class SeenPair { private Object s1; private Object s2; private SeenPair(Object s1, Object s2) { this.s1 = s1; this.s2 = s2; } public boolean equals(Object o) { + if (!(o instanceof SeenPair)) return false; return this.s1 == ((SeenPair)o).s1 && this.s2 == ((SeenPair)o).s2; } public int hashCode() { diff --git a/lang/java/avro/src/main/java/org/apache/avro/SchemaBuilder.java b/lang/java/avro/src/main/java/org/apache/avro/SchemaBuilder.java index f1a1faa1515..ce038d4b448 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/SchemaBuilder.java +++ b/lang/java/avro/src/main/java/org/apache/avro/SchemaBuilder.java @@ -2587,6 +2587,10 @@ private static JsonNode toJsonNode(Object o) { s = new String(data, "ISO-8859-1"); char[] quoted = JsonStringEncoder.getInstance().quoteAsString(s); s = "\"" + new String(quoted) + "\""; + } else if (o instanceof byte[]) { + s = new String((byte[]) o, "ISO-8859-1"); + char[] quoted = JsonStringEncoder.getInstance().quoteAsString(s); + s = '\"' + new String(quoted) + '\"'; } else { s = GenericData.get().toString(o); } diff --git a/lang/java/avro/src/main/java/org/apache/avro/file/BZip2Codec.java b/lang/java/avro/src/main/java/org/apache/avro/file/BZip2Codec.java index 8dccfc33ffd..ca90d6e3b84 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/file/BZip2Codec.java +++ b/lang/java/avro/src/main/java/org/apache/avro/file/BZip2Codec.java @@ -87,7 +87,7 @@ public ByteBuffer decompress(ByteBuffer compressedData) throws IOException { public boolean equals(Object obj) { if (this == obj) return true; - if (getClass() != obj.getClass()) + if (obj == null || obj.getClass() != getClass()) return false; return true; } diff --git a/lang/java/avro/src/main/java/org/apache/avro/file/DeflateCodec.java b/lang/java/avro/src/main/java/org/apache/avro/file/DeflateCodec.java index f8f6ac42a5d..bfe9e6d0ad7 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/file/DeflateCodec.java +++ b/lang/java/avro/src/main/java/org/apache/avro/file/DeflateCodec.java @@ -132,7 +132,7 @@ public int hashCode() { public boolean equals(Object obj) { if (this == obj) return true; - if (getClass() != obj.getClass()) + if (obj == null || obj.getClass() != getClass()) return false; DeflateCodec other = (DeflateCodec)obj; return (this.nowrap == other.nowrap); diff --git a/lang/java/avro/src/main/java/org/apache/avro/file/NullCodec.java b/lang/java/avro/src/main/java/org/apache/avro/file/NullCodec.java index e95f699e754..bc07f146ba7 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/file/NullCodec.java +++ b/lang/java/avro/src/main/java/org/apache/avro/file/NullCodec.java @@ -54,7 +54,7 @@ public ByteBuffer decompress(ByteBuffer data) throws IOException { public boolean equals(Object other) { if (this == other) return true; - return (this.getClass() == other.getClass()); + return (other != null && other.getClass() == getClass()); } @Override diff --git a/lang/java/avro/src/main/java/org/apache/avro/file/SnappyCodec.java b/lang/java/avro/src/main/java/org/apache/avro/file/SnappyCodec.java index 1a5d252692f..6206c8d24c8 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/file/SnappyCodec.java +++ b/lang/java/avro/src/main/java/org/apache/avro/file/SnappyCodec.java @@ -75,7 +75,7 @@ public ByteBuffer decompress(ByteBuffer in) throws IOException { public boolean equals(Object obj) { if (this == obj) return true; - if (getClass() != obj.getClass()) + if (obj == null || obj.getClass() != getClass()) return false; return true; } diff --git a/lang/java/avro/src/main/java/org/apache/avro/file/XZCodec.java b/lang/java/avro/src/main/java/org/apache/avro/file/XZCodec.java index 7677b3fb676..23aa8308aa0 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/file/XZCodec.java +++ b/lang/java/avro/src/main/java/org/apache/avro/file/XZCodec.java @@ -109,7 +109,7 @@ public int hashCode() { public boolean equals(Object obj) { if (this == obj) return true; - if (getClass() != obj.getClass()) + if (obj == null || obj.getClass() != getClass()) return false; XZCodec other = (XZCodec)obj; return (this.compressionLevel == other.compressionLevel); diff --git a/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificRecordBase.java b/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificRecordBase.java index baedeb872f7..20d3dc331da 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificRecordBase.java +++ b/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificRecordBase.java @@ -50,7 +50,7 @@ public Object get(String fieldName) { return get(getSchema().getField(fieldName).pos()); } - public Conversion getConverion(String fieldName) { + public Conversion getConversion(String fieldName) { return getConversion(getSchema().getField(fieldName).pos()); } diff --git a/lang/java/avro/src/main/java/org/apache/avro/util/WeakIdentityHashMap.java b/lang/java/avro/src/main/java/org/apache/avro/util/WeakIdentityHashMap.java index a22708a72ef..6958798c407 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/util/WeakIdentityHashMap.java +++ b/lang/java/avro/src/main/java/org/apache/avro/util/WeakIdentityHashMap.java @@ -96,6 +96,9 @@ public Set keySet() { } public boolean equals(Object o) { + if (!(o instanceof WeakIdentityHashMap)) { + return false; + } return backingStore.equals(((WeakIdentityHashMap)o).backingStore); } @@ -159,6 +162,9 @@ public boolean equals(Object o) { if (this == o) { return true; } + if (!(o instanceof WeakIdentityHashMap.IdentityWeakReference)) { + return false; + } IdentityWeakReference ref = (IdentityWeakReference)o; if (this.get() == ref.get()) { return true; diff --git a/lang/java/avro/src/main/java/org/apache/avro/util/internal/JacksonUtils.java b/lang/java/avro/src/main/java/org/apache/avro/util/internal/JacksonUtils.java index ca98e4c540d..49b939e729a 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/util/internal/JacksonUtils.java +++ b/lang/java/avro/src/main/java/org/apache/avro/util/internal/JacksonUtils.java @@ -120,7 +120,8 @@ public static Object toObject(JsonNode jsonNode, Schema schema) { if (schema == null || schema.getType().equals(Schema.Type.STRING) || schema.getType().equals(Schema.Type.ENUM)) { return jsonNode.asText(); - } else if (schema.getType().equals(Schema.Type.BYTES)) { + } else if (schema.getType().equals(Schema.Type.BYTES) + || schema.getType().equals(Schema.Type.FIXED)) { try { return jsonNode.getTextValue().getBytes(BYTES_CHARSET); } catch (UnsupportedEncodingException e) { diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestFixed.java b/lang/java/avro/src/test/java/org/apache/avro/TestFixed.java index 14ff5cef49a..5b69d1ef93b 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/TestFixed.java +++ b/lang/java/avro/src/test/java/org/apache/avro/TestFixed.java @@ -1,21 +1,5 @@ -/* - * Copyright 2017 The Apache Software Foundation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package org.apache.avro; -import java.nio.ByteBuffer; import org.junit.Assert; import org.junit.Test; @@ -26,9 +10,10 @@ public class TestFixed { public void testFixedDefaultValueDrop() { Schema md5 = SchemaBuilder.builder().fixed("MD5").size(16); Schema frec = SchemaBuilder.builder().record("test") - .fields().name("hash").type(md5).withDefault(ByteBuffer.wrap(new byte[16])).endRecord(); + .fields().name("hash").type(md5).withDefault(new byte[16]).endRecord(); Schema.Field field = frec.getField("hash"); Assert.assertNotNull(field.defaultVal()); + Assert.assertArrayEquals(new byte[16], (byte[]) field.defaultVal()); } -} +} \ No newline at end of file diff --git a/lang/java/compiler/src/test/idl/input/cycle.avdl b/lang/java/compiler/src/test/idl/input/cycle.avdl index f4344316024..07c967594f6 100644 --- a/lang/java/compiler/src/test/idl/input/cycle.avdl +++ b/lang/java/compiler/src/test/idl/input/cycle.avdl @@ -7,7 +7,7 @@ protocol Cycle { } record Method { - string declaringClass; + string @testAttribute("testValue") declaringClass; string methodName; } diff --git a/lang/java/compiler/src/test/idl/output/cycle.avpr b/lang/java/compiler/src/test/idl/output/cycle.avpr index 53658afd00e..190c36b3c7d 100644 --- a/lang/java/compiler/src/test/idl/output/cycle.avpr +++ b/lang/java/compiler/src/test/idl/output/cycle.avpr @@ -22,7 +22,8 @@ "name" : "Method", "fields" : [ { "name" : "declaringClass", - "type" : "string" + "type" : "string", + "testAttribute":"testValue" }, { "name" : "methodName", "type" : "string" diff --git a/lang/java/ipc/src/test/java/org/apache/avro/ipc/TestNettyServerWithCallbacks.java b/lang/java/ipc/src/test/java/org/apache/avro/ipc/TestNettyServerWithCallbacks.java index 3a9e158682b..6ed898a67f0 100644 --- a/lang/java/ipc/src/test/java/org/apache/avro/ipc/TestNettyServerWithCallbacks.java +++ b/lang/java/ipc/src/test/java/org/apache/avro/ipc/TestNettyServerWithCallbacks.java @@ -25,6 +25,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -370,8 +371,8 @@ public void cancelPendingRequestsOnTransceiverClose() throws Exception { } } - @Test - public void cancelPendingRequestsAfterChannelCloseByServerShutdown() throws Exception { + @Test(timeout = 20000) + public void cancelPendingRequestsAfterChannelCloseByServerShutdown() throws Throwable { // The purpose of this test is to verify that a client doesn't stay // blocked when a server is unexpectedly killed (or when for some // other reason the channel is suddenly closed) while the server @@ -381,7 +382,7 @@ public void cancelPendingRequestsAfterChannelCloseByServerShutdown() throws Exce // Start up a second server so that closing the server doesn't // interfere with the other unit tests: BlockingSimpleImpl blockingSimpleImpl = new BlockingSimpleImpl(); - Server server2 = new NettyServer(new SpecificResponder(Simple.class, + final Server server2 = new NettyServer(new SpecificResponder(Simple.class, blockingSimpleImpl), new InetSocketAddress(0)); server2.start(); @@ -404,7 +405,8 @@ public void cancelPendingRequestsAfterChannelCloseByServerShutdown() throws Exce // Acquire the run permit, to avoid that the server method returns immediately blockingSimpleImpl.acquireRunPermit(); - Thread t = new Thread(new Runnable() { + // Start client call + Future clientFuture = Executors.newSingleThreadExecutor().submit(new Runnable() { @Override public void run() { try { @@ -416,23 +418,30 @@ public void run() { } }); - // Start client call - t.start(); - // Wait until method is entered on the server side blockingSimpleImpl.acquireEnterPermit(); // The server side method is now blocked waiting on the run permit // (= is busy handling the request) - // Stop the server - server2.close(); + // Stop the server in a separate thread as it blocks the actual thread until the server side + // method is running + new Thread(new Runnable() { + @Override + public void run() { + server2.close(); + } + }).start(); // With the server gone, we expect the client to get some exception and exit - // Wait for client thread to exit - t.join(10000); - - Assert.assertFalse("Client request should not be blocked on server shutdown", t.isAlive()); + // Wait for the client call to exit + try { + clientFuture.get(10, TimeUnit.SECONDS); + } catch (ExecutionException e) { + throw e.getCause(); + } catch (TimeoutException e) { + Assert.fail("Client request should not be blocked on server shutdown"); + } } finally { blockingSimpleImpl.releaseRunPermit(); diff --git a/lang/java/pom.xml b/lang/java/pom.xml index 8afcefd5de4..70c86931644 100644 --- a/lang/java/pom.xml +++ b/lang/java/pom.xml @@ -43,51 +43,49 @@ -Dhadoop.version=1 or leave unspecified to build against Hadoop 2 --> 1.2.1 - 2.5.1 + 2.7.3 1.9.13 6.1.26 2.5-20081211 - 4.7 - 4.11 - 3.5.13.Final - 2.7 - 2.5.0 - 0.9.1 - 1.7.7 - 1.1.1.3 + 5.0.3 + 4.12 + 3.10.6.Final + 2.8 + 2.6.1 + 0.9.3 + 1.7.22 + 1.1.2.6 1.7 - 2.0.10 - 1.9.0 - 1.2 - 1.9 - 1.8.1 - 3.1 + 2.0.11 + 1.10.0 + 1.3.1 + 1.10 + 1.13 2.6 - 1.1.1 - 1.5 - 3.2 + 1.2 + 1.6 + 3.4 1.3 - 3.1 - 2.7 + 2.9.7 11.0.2 1.3.9-1 - 2.5.3 - 3.1 - 1.3.2 - 2.5 + 3.2.0 + 3.6.0 + 1.5.0 + 2.6 2.6 - 2.9.1 - 3.2 - 3.3 - 3.3 - 2.3 - 2.17 + 2.10.4 + 3.5 + 3.6 + 3.5 + 2.4 + 2.19.1 1.2.1 1.7.1 - 2.2 + 2.4 diff --git a/lang/java/tools/pom.xml b/lang/java/tools/pom.xml index 18e01558a61..11bac66ae88 100644 --- a/lang/java/tools/pom.xml +++ b/lang/java/tools/pom.xml @@ -220,11 +220,6 @@ commons-logging ${commons-logging.version} - - commons-httpclient - commons-httpclient - ${commons-httpclient.version} - ${project.groupId} trevni-core diff --git a/lang/java/tools/src/main/java/org/apache/avro/tool/ConcatTool.java b/lang/java/tools/src/main/java/org/apache/avro/tool/ConcatTool.java index 60267968fde..e782321822c 100644 --- a/lang/java/tools/src/main/java/org/apache/avro/tool/ConcatTool.java +++ b/lang/java/tools/src/main/java/org/apache/avro/tool/ConcatTool.java @@ -17,9 +17,11 @@ */ package org.apache.avro.tool; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.PrintStream; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -33,6 +35,7 @@ import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.generic.GenericDatumWriter; import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.fs.Path; /** * Tool to concatenate avro files with the same schema and non-reserved @@ -65,7 +68,7 @@ public int run(InputStream in, PrintStream out, PrintStream err, Map metadata = new TreeMap(); String inputCodec = null; - for (String inFile : args) { + for (String inFile : expandsInputFiles(args)) { InputStream input = Util.fileOrStdin(inFile, in); DataFileStream reader = new DataFileStream( input, new GenericDatumReader()); @@ -124,6 +127,24 @@ public int run(InputStream in, PrintStream out, PrintStream err, return 0; } + /** Processes a list of input files to expand directories if needed. */ + private static List expandsInputFiles(List args) throws IOException { + List files = new ArrayList(); + + for (String arg : args) { + if (arg.equals("-")) { + files.add(arg); + } else { + List paths = Util.getFiles(arg); + for (Path path : paths) { + files.add(path.toString()); + } + } + } + + return files; + } + private void printHelp(PrintStream out) { out.println("concat [input-file...] output-file"); out.println(); @@ -136,8 +157,9 @@ private void printHelp(PrintStream out) { out.println(" 3 if the codecs don't match"); out.println("If no input files are given stdin will be used. The tool"); out.println("0 on success. A dash ('-') can be given as an input file"); - out.println("to use stdin, and as an output file to use stdout."); - + out.println("to use stdin, and as an output file to use stdout. If a directory"); + out.println("is given as an input-file all the files within this directory"); + out.println("are used."); } @Override diff --git a/lang/java/tools/src/main/java/org/apache/avro/tool/Util.java b/lang/java/tools/src/main/java/org/apache/avro/tool/Util.java index 708bb41c40f..9f1cae184b9 100644 --- a/lang/java/tools/src/main/java/org/apache/avro/tool/Util.java +++ b/lang/java/tools/src/main/java/org/apache/avro/tool/Util.java @@ -22,6 +22,7 @@ import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -168,42 +169,52 @@ static Schema parseSchemaFromFS(String filename) throws IOException { } } - /**If pathname is a file, this method returns a list with a single absolute Path to that file, - * if pathname is a directory, this method returns a list of Pathes to all the files within - * this directory. - * Only files inside that directory are included, no subdirectories or files in subdirectories - * will be added. + /** + * If pathname is a file, this method returns a list with a single absolute Path to that file. + * If pathname is a directory, this method returns a list of Pathes to all the files within + * this directory. Only files inside that directory are included, no subdirectories or files + * in subdirectories will be added. + * If pathname is a glob pattern, all files matching the pattern are included. + * * The List is sorted alphabetically. - * @param fileOrDirName filename or directoryname + * @param fileOrDirName filename, directoryname or a glob pattern * @return A Path List * @throws IOException */ - static List getFiles(String fileOrDirName) - throws IOException { + static List getFiles(String fileOrDirName) throws IOException { List pathList = new ArrayList(); Path path = new Path(fileOrDirName); FileSystem fs = path.getFileSystem(new Configuration()); if (fs.isFile(path)) { pathList.add(path); - } - else if (fs.getFileStatus(path).isDir()) { + } else if (fs.isDirectory(path)) { for (FileStatus status : fs.listStatus(path)) { if(!status.isDir()) { pathList.add(status.getPath()); } } + } else { + FileStatus[] fileStatuses = fs.globStatus(path); + if (fileStatuses != null) { + for (FileStatus status : fileStatuses) { + pathList.add(status.getPath()); + } + } else { + throw new FileNotFoundException(fileOrDirName); + } } Collections.sort(pathList); return pathList; } /** - * This method returns a list which contains a path to every given file - * in the input and a path to every file inside a given directory. + * Concatenate the result of {@link #getFiles(String)} applied to all file or directory names. * The list is sorted alphabetically and contains no subdirectories or files within those. - * @param fileOrDirNames A list of filenames and directorynames - * @return A list of Pathes, one for each file + * + * The list is sorted alphabetically. + * @param fileOrDirNames A list of filenames, directorynames or glob patterns + * @return A list of Paths, one for each file * @throws IOException */ static List getFiles(List fileOrDirNames) diff --git a/lang/java/tools/src/main/resources/log4j.properties b/lang/java/tools/src/main/resources/log4j.properties new file mode 100644 index 00000000000..41894f65256 --- /dev/null +++ b/lang/java/tools/src/main/resources/log4j.properties @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log4j.rootLogger=INFO, console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n diff --git a/lang/java/tools/src/test/java/org/apache/avro/tool/TestCatTool.java b/lang/java/tools/src/test/java/org/apache/avro/tool/TestCatTool.java index 312bd76d778..39e45de46df 100644 --- a/lang/java/tools/src/test/java/org/apache/avro/tool/TestCatTool.java +++ b/lang/java/tools/src/test/java/org/apache/avro/tool/TestCatTool.java @@ -180,6 +180,20 @@ public void testCat() throws Exception { args); assertEquals(0, returnCode); assertEquals(LIMIT_WITHIN_INPUT_BOUNDS, numRowsInFile(output)); + +// glob input + args = asList( + new File(input1.getParentFile(), "/*").getAbsolutePath(), + output.getAbsolutePath(), + "--offset" , String.valueOf(OFFSET), + "--limit" , String.valueOf(LIMIT_WITHIN_INPUT_BOUNDS)); + returnCode = new CatTool().run( + System.in, + System.out, + System.err, + args); + assertEquals(0, returnCode); + assertEquals(LIMIT_WITHIN_INPUT_BOUNDS, numRowsInFile(output)); } diff --git a/lang/java/tools/src/test/java/org/apache/avro/tool/TestConcatTool.java b/lang/java/tools/src/test/java/org/apache/avro/tool/TestConcatTool.java index af31ccbca4a..6fdbddf9a41 100644 --- a/lang/java/tools/src/test/java/org/apache/avro/tool/TestConcatTool.java +++ b/lang/java/tools/src/test/java/org/apache/avro/tool/TestConcatTool.java @@ -25,7 +25,9 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.PrintStream; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; @@ -107,6 +109,83 @@ private int numRowsInFile(File output) throws Exception { return rowcount; } + @Test + public void testDirConcat() throws Exception { + Map metadata = new HashMap(); + + File dir = AvroTestUtil.tempDirectory(getClass(), "input"); + + for (int i = 0; i < 3; i++) { + String filename = "input" + i + ".avro"; + File input = generateData(filename, Type.STRING, metadata, DEFLATE); + boolean ok = input.renameTo(new File(dir, input.getName())); + assertTrue(ok); + } + + File output = AvroTestUtil.tempFile(getClass(), "default-output.avro"); + output.deleteOnExit(); + + List args = asList( + dir.getAbsolutePath(), + output.getAbsolutePath()); + int returnCode = new ConcatTool().run( + System.in, + System.out, + System.err, + args); + assertEquals(0, returnCode); + + assertEquals(ROWS_IN_INPUT_FILES * 3, numRowsInFile(output)); + } + + @Test + public void testGlobPatternConcat() throws Exception { + Map metadata = new HashMap(); + + File dir = AvroTestUtil.tempDirectory(getClass(), "input"); + + for (int i = 0; i < 3; i++) { + String filename = "input" + i + ".avro"; + File input = generateData(filename, Type.STRING, metadata, DEFLATE); + boolean ok = input.renameTo(new File(dir, input.getName())); + assertTrue(ok); + } + + File output = AvroTestUtil.tempFile(getClass(), "default-output.avro"); + output.deleteOnExit(); + + List args = asList( + new File(dir, "/*").getAbsolutePath(), + output.getAbsolutePath()); + int returnCode = new ConcatTool().run( + System.in, + System.out, + System.err, + args); + assertEquals(0, returnCode); + + assertEquals(ROWS_IN_INPUT_FILES * 3, numRowsInFile(output)); + } + + @Test(expected = FileNotFoundException.class) + public void testFileDoesNotExist() throws Exception { + Map metadata = new HashMap(); + + File dir = AvroTestUtil.tempDirectory(getClass(), "input"); + + File output = AvroTestUtil.tempFile(getClass(), "default-output.avro"); + output.deleteOnExit(); + + List args = asList( + new File(dir, "/doNotExist").getAbsolutePath(), + output.getAbsolutePath()); + new ConcatTool().run( + System.in, + System.out, + System.err, + args); + } + @Test public void testConcat() throws Exception { Map metadata = new HashMap(); diff --git a/lang/ruby/Manifest b/lang/ruby/Manifest index 23220bb9bf5..3edd7cff706 100644 --- a/lang/ruby/Manifest +++ b/lang/ruby/Manifest @@ -12,6 +12,7 @@ lib/avro/ipc.rb lib/avro/protocol.rb lib/avro/schema.rb lib/avro/schema_normalization.rb +lib/avro/schema_validator.rb test/case_finder.rb test/random_data.rb test/sample_ipc_client.rb @@ -25,5 +26,6 @@ test/test_io.rb test/test_protocol.rb test/test_schema.rb test/test_schema_normalization.rb +test/test_schema_validator.rb test/test_socket_transport.rb test/tool.rb diff --git a/lang/ruby/lib/avro.rb b/lang/ruby/lib/avro.rb index c419ab1e6b3..1293f0f15d3 100644 --- a/lang/ruby/lib/avro.rb +++ b/lang/ruby/lib/avro.rb @@ -40,3 +40,4 @@ def initialize(schm=nil, datum=nil, msg=nil) require 'avro/protocol' require 'avro/ipc' require 'avro/schema_normalization' +require 'avro/schema_validator' diff --git a/lang/ruby/lib/avro/schema.rb b/lang/ruby/lib/avro/schema.rb index 8e345e34f7e..477528e5a3a 100644 --- a/lang/ruby/lib/avro/schema.rb +++ b/lang/ruby/lib/avro/schema.rb @@ -92,39 +92,10 @@ def self.real_parse(json_obj, names=nil, default_namespace=nil) # Determine if a ruby datum is an instance of a schema def self.validate(expected_schema, datum) - case expected_schema.type_sym - when :null - datum.nil? - when :boolean - datum == true || datum == false - when :string, :bytes - datum.is_a? String - when :int - (datum.is_a?(Fixnum) || datum.is_a?(Bignum)) && - (INT_MIN_VALUE <= datum) && (datum <= INT_MAX_VALUE) - when :long - (datum.is_a?(Fixnum) || datum.is_a?(Bignum)) && - (LONG_MIN_VALUE <= datum) && (datum <= LONG_MAX_VALUE) - when :float, :double - datum.is_a?(Float) || datum.is_a?(Fixnum) || datum.is_a?(Bignum) - when :fixed - datum.is_a?(String) && datum.bytesize == expected_schema.size - when :enum - expected_schema.symbols.include? datum - when :array - datum.is_a?(Array) && - datum.all?{|d| validate(expected_schema.items, d) } - when :map - datum.keys.all?{|k| k.is_a? String } && - datum.values.all?{|v| validate(expected_schema.values, v) } - when :union - expected_schema.schemas.any?{|s| validate(s, datum) } - when :record, :error, :request - datum.is_a?(Hash) && - expected_schema.fields.all?{|f| validate(f.type, datum[f.name]) } - else - raise "you suck #{expected_schema.inspect} is not allowed." - end + SchemaValidator.validate!(expected_schema, datum) + true + rescue SchemaValidator::ValidationError + false end def initialize(type) @@ -347,7 +318,7 @@ class FixedSchema < NamedSchema attr_reader :size def initialize(name, space, size, names=nil) # Ensure valid cto args - unless size.is_a?(Fixnum) || size.is_a?(Bignum) + unless size.is_a?(Integer) raise AvroError, 'Fixed Schema requires a valid integer for size property.' end super(:fixed, name, space, names) diff --git a/lang/ruby/lib/avro/schema_validator.rb b/lang/ruby/lib/avro/schema_validator.rb new file mode 100644 index 00000000000..ca4f7f6f545 --- /dev/null +++ b/lang/ruby/lib/avro/schema_validator.rb @@ -0,0 +1,196 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module Avro + class SchemaValidator + ROOT_IDENTIFIER = '.'.freeze + PATH_SEPARATOR = '.'.freeze + INT_RANGE = Schema::INT_MIN_VALUE..Schema::INT_MAX_VALUE + LONG_RANGE = Schema::LONG_MIN_VALUE..Schema::LONG_MAX_VALUE + COMPLEX_TYPES = [:array, :error, :map, :record, :request] + + class Result + attr_reader :errors + + def initialize + @errors = [] + end + + def <<(error) + @errors << error + end + + def add_error(path, message) + self << "at #{path} #{message}" + end + + def failure? + @errors.any? + end + + def to_s + errors.join("\n") + end + end + + class ValidationError < StandardError + attr_reader :result + + def initialize(result = Result.new) + @result = result + super + end + + def to_s + result.to_s + end + end + + TypeMismatchError = Class.new(ValidationError) + + class << self + def validate!(expected_schema, datum) + result = Result.new + validate_recursive(expected_schema, datum, ROOT_IDENTIFIER, result) + fail ValidationError, result if result.failure? + result + end + + private + + def validate_recursive(expected_schema, datum, path, result) + case expected_schema.type_sym + when :null + fail TypeMismatchError unless datum.nil? + when :boolean + fail TypeMismatchError unless [true, false].include?(datum) + when :string, :bytes + fail TypeMismatchError unless datum.is_a?(String) + when :int + fail TypeMismatchError unless datum.is_a?(Integer) + result.add_error(path, "out of bound value #{datum}") unless INT_RANGE.cover?(datum) + when :long + fail TypeMismatchError unless datum.is_a?(Integer) + result.add_error(path, "out of bound value #{datum}") unless LONG_RANGE.cover?(datum) + when :float, :double + fail TypeMismatchError unless [Float, Integer].any?(&datum.method(:is_a?)) + when :fixed + if datum.is_a? String + message = "expected fixed with size #{expected_schema.size}, got \"#{datum}\" with size #{datum.size}" + result.add_error(path, message) unless datum.bytesize == expected_schema.size + else + result.add_error(path, "expected fixed with size #{expected_schema.size}, got #{actual_value_message(datum)}") + end + when :enum + message = "expected enum with values #{expected_schema.symbols}, got #{actual_value_message(datum)}" + result.add_error(path, message) unless expected_schema.symbols.include?(datum) + when :array + validate_array(expected_schema, datum, path, result) + when :map + validate_map(expected_schema, datum, path, result) + when :union + validate_union(expected_schema, datum, path, result) + when :record, :error, :request + fail TypeMismatchError unless datum.is_a?(Hash) + expected_schema.fields.each do |field| + deeper_path = deeper_path_for_hash(field.name, path) + validate_recursive(field.type, datum[field.name], deeper_path, result) + end + else + fail "Unexpected schema type #{expected_schema.type_sym} #{expected_schema.inspect}" + end + rescue TypeMismatchError + result.add_error(path, "expected type #{expected_schema.type_sym}, got #{actual_value_message(datum)}") + end + + def validate_array(expected_schema, datum, path, result) + fail TypeMismatchError unless datum.is_a?(Array) + datum.each_with_index do |d, i| + validate_recursive(expected_schema.items, d, path + "[#{i}]", result) + end + end + + def validate_map(expected_schema, datum, path, result) + datum.keys.each do |k| + result.add_error(path, "unexpected key type '#{ruby_to_avro_type(k.class)}' in map") unless k.is_a?(String) + end + datum.each do |k, v| + deeper_path = deeper_path_for_hash(k, path) + validate_recursive(expected_schema.values, v, deeper_path, result) + end + end + + def validate_union(expected_schema, datum, path, result) + if expected_schema.schemas.size == 1 + validate_recursive(expected_schema.schemas.first, datum, path, result) + return + end + types_and_results = validate_possible_types(datum, expected_schema, path) + failures, successes = types_and_results.partition { |r| r[:result].failure? } + return if successes.any? + complex_type_failed = failures.detect { |r| COMPLEX_TYPES.include?(r[:type]) } + if complex_type_failed + complex_type_failed[:result].errors.each { |error| result << error } + else + types = expected_schema.schemas.map { |s| "'#{s.type_sym}'" }.join(', ') + result.add_error(path, "expected union of [#{types}], got #{actual_value_message(datum)}") + end + end + + def validate_possible_types(datum, expected_schema, path) + expected_schema.schemas.map do |schema| + result = Result.new + validate_recursive(schema, datum, path, result) + { type: schema.type_sym, result: result } + end + end + + def deeper_path_for_hash(sub_key, path) + "#{path}#{PATH_SEPARATOR}#{sub_key}".squeeze(PATH_SEPARATOR) + end + + private + + def actual_value_message(value) + avro_type = if value.class == Integer + ruby_integer_to_avro_type(value) + else + ruby_to_avro_type(value.class) + end + if value.nil? + avro_type + else + "#{avro_type} with value #{value.inspect}" + end + end + + def ruby_to_avro_type(ruby_class) + { + NilClass => 'null', + String => 'string', + Fixnum => 'int', + Bignum => 'long', + Float => 'float', + Hash => 'record' + }.fetch(ruby_class, ruby_class) + end + + def ruby_integer_to_avro_type(value) + INT_RANGE.cover?(value) ? 'int' : 'long' + end + end + end +end diff --git a/lang/ruby/test/test_schema.rb b/lang/ruby/test/test_schema.rb index 164330744c8..a0118238e79 100644 --- a/lang/ruby/test/test_schema.rb +++ b/lang/ruby/test/test_schema.rb @@ -27,13 +27,13 @@ def test_default_namespace ]} SCHEMA - assert_equal schema.name, 'OuterRecord' - assert_equal schema.fullname, 'OuterRecord' + assert_equal 'OuterRecord', schema.name + assert_equal 'OuterRecord', schema.fullname assert_nil schema.namespace schema.fields.each do |field| - assert_equal field.type.name, 'InnerRecord' - assert_equal field.type.fullname, 'InnerRecord' + assert_equal 'InnerRecord', field.type.name + assert_equal 'InnerRecord', field.type.fullname assert_nil field.type.namespace end end @@ -50,13 +50,13 @@ def test_inherited_namespace ]} SCHEMA - assert_equal schema.name, 'OuterRecord' - assert_equal schema.fullname, 'my.name.space.OuterRecord' - assert_equal schema.namespace, 'my.name.space' + assert_equal 'OuterRecord', schema.name + assert_equal 'my.name.space.OuterRecord', schema.fullname + assert_equal 'my.name.space', schema.namespace schema.fields.each do |field| - assert_equal field.type.name, 'InnerRecord' - assert_equal field.type.fullname, 'my.name.space.InnerRecord' - assert_equal field.type.namespace, 'my.name.space' + assert_equal 'InnerRecord', field.type.name + assert_equal 'my.name.space.InnerRecord', field.type.fullname + assert_equal 'my.name.space', field.type.namespace end end @@ -71,13 +71,13 @@ def test_inherited_namespace_from_dotted_name ]} SCHEMA - assert_equal schema.name, 'OuterRecord' - assert_equal schema.fullname, 'my.name.space.OuterRecord' - assert_equal schema.namespace, 'my.name.space' + assert_equal 'OuterRecord', schema.name + assert_equal 'my.name.space.OuterRecord', schema.fullname + assert_equal 'my.name.space', schema.namespace schema.fields.each do |field| - assert_equal field.type.name, 'InnerEnum' - assert_equal field.type.fullname, 'my.name.space.InnerEnum' - assert_equal field.type.namespace, 'my.name.space' + assert_equal 'InnerEnum', field.type.name + assert_equal 'my.name.space.InnerEnum', field.type.fullname + assert_equal 'my.name.space', field.type.namespace end end @@ -96,18 +96,18 @@ def test_nested_namespaces ]} SCHEMA - assert_equal schema.name, 'OuterRecord' - assert_equal schema.fullname, 'outer.OuterRecord' - assert_equal schema.namespace, 'outer' + assert_equal 'OuterRecord', schema.name + assert_equal 'outer.OuterRecord', schema.fullname + assert_equal 'outer', schema.namespace middle = schema.fields.first.type - assert_equal middle.name, 'MiddleRecord' - assert_equal middle.fullname, 'middle.MiddleRecord' - assert_equal middle.namespace, 'middle' + assert_equal 'MiddleRecord', middle.name + assert_equal 'middle.MiddleRecord', middle.fullname + assert_equal 'middle', middle.namespace inner = middle.fields.first.type - assert_equal inner.name, 'InnerRecord' - assert_equal inner.fullname, 'middle.InnerRecord' - assert_equal inner.namespace, 'middle' - assert_equal inner.fields.first.type, middle + assert_equal 'InnerRecord', inner.name + assert_equal 'middle.InnerRecord', inner.fullname + assert_equal 'middle', inner.namespace + assert_equal middle, inner.fields.first.type end def test_to_avro_includes_namespaces @@ -120,7 +120,7 @@ def test_to_avro_includes_namespaces ]} SCHEMA - assert_equal schema.to_avro, { + assert_equal({ 'type' => 'record', 'name' => 'OuterRecord', 'namespace' => 'my.name.space', 'fields' => [ {'name' => 'definition', 'type' => { @@ -129,7 +129,7 @@ def test_to_avro_includes_namespaces }}, {'name' => 'reference', 'type' => 'my.name.space.InnerFixed'} ] - } + }, schema.to_avro) end def test_unknown_named_type diff --git a/lang/ruby/test/test_schema_validator.rb b/lang/ruby/test/test_schema_validator.rb new file mode 100644 index 00000000000..93f2ca85b95 --- /dev/null +++ b/lang/ruby/test/test_schema_validator.rb @@ -0,0 +1,402 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'test_help' + +class TestSchema < Test::Unit::TestCase + def validate!(schema, value) + Avro::SchemaValidator.validate!(schema, value) + end + + def hash_to_schema(hash) + Avro::Schema.parse(hash.to_json) + end + + def assert_failed_validation(messages) + error = assert_raise(Avro::SchemaValidator::ValidationError) { yield } + + assert_messages = [messages].flatten + result_errors = error.result.errors + assert_messages.each do |message| + assert(result_errors.include?(message), "expected '#{message}' to be in '#{result_errors}'") + end + assert_equal(assert_messages.size, result_errors.size) + end + + def assert_valid_schema(schema, valid, invalid) + valid.each do |value| + assert_nothing_raised { Avro::SchemaValidator.validate!(schema, value) } + end + + invalid.each do |value| + assert_raise { Avro::SchemaValidator.validate!(schema, value) } + end + end + + def test_validate_nil + schema = hash_to_schema(type: 'null', name: 'name') + + assert_nothing_raised { validate!(schema, nil) } + + assert_failed_validation('at . expected type null, got int with value 1') do + validate!(schema, 1) + end + end + + def test_validate_boolean + schema = hash_to_schema(type: 'boolean', name: 'name') + + assert_nothing_raised { validate!(schema, true) } + assert_nothing_raised { validate!(schema, false) } + + assert_failed_validation('at . expected type boolean, got int with value 1') do + validate!(schema, 1) + end + + assert_failed_validation('at . expected type boolean, got null') do + validate!(schema, nil) + end + end + + def test_fixed_size_string + schema = hash_to_schema(type: 'fixed', name: 'some', size: 3) + + assert_nothing_raised { validate!(schema, 'baf') } + + assert_failed_validation('at . expected fixed with size 3, got "some" with size 4') do + validate!(schema, 'some') + end + + assert_failed_validation('at . expected fixed with size 3, got null') do + validate!(schema, nil) + end + end + + def test_original_validate_nil + schema = hash_to_schema(type: 'null', name: 'name') + + assert_valid_schema(schema, [nil], ['something']) + end + + def test_original_validate_boolean + schema = hash_to_schema(type: 'boolean', name: 'name') + + assert_valid_schema(schema, [true, false], [nil, 1]) + end + + def test_validate_string + schema = hash_to_schema(type: 'string', name: 'name') + + assert_valid_schema(schema, ['string'], [nil, 1]) + end + + def test_validate_bytes + schema = hash_to_schema(type: 'bytes', name: 'name') + + assert_valid_schema(schema, ['string'], [nil, 1]) + end + + def test_validate_int + schema = hash_to_schema(type: 'int', name: 'name') + + assert_valid_schema( + schema, + [Avro::Schema::INT_MIN_VALUE, Avro::Schema::INT_MAX_VALUE, 1], + [Avro::Schema::LONG_MIN_VALUE, Avro::Schema::LONG_MAX_VALUE, 'string'] + ) + assert_failed_validation('at . out of bound value 9223372036854775807') do + validate!(schema, Avro::Schema::LONG_MAX_VALUE) + end + end + + def test_validate_long + schema = hash_to_schema(type: 'long', name: 'name') + + assert_valid_schema(schema, [Avro::Schema::LONG_MIN_VALUE, Avro::Schema::LONG_MAX_VALUE, 1], [1.1, 'string']) + end + + def test_validate_float + schema = hash_to_schema(type: 'float', name: 'name') + + assert_valid_schema(schema, [1.1, 1, Avro::Schema::LONG_MAX_VALUE], ['string']) + end + + def test_validate_double + schema = hash_to_schema(type: 'double', name: 'name') + + assert_valid_schema(schema, [1.1, 1, Avro::Schema::LONG_MAX_VALUE], ['string']) + end + + def test_validate_fixed + schema = hash_to_schema(type: 'fixed', name: 'name', size: 3) + + assert_valid_schema(schema, ['abc'], ['ab', 1, 1.1, true]) + end + + def test_validate_original_num + schema = hash_to_schema(type: 'enum', name: 'name', symbols: %w(a b)) + + assert_valid_schema(schema, ['a', 'b'], ['c']) + end + + def test_validate_record + schema = hash_to_schema(type: 'record', name: 'name', fields: [{ type: 'null', name: 'sub' }]) + + assert_valid_schema(schema, [{ 'sub' => nil }], [{ 'sub' => 1 }]) + end + + def test_validate_shallow_record + schema = hash_to_schema( + type: 'record', name: 'name', fields: [{ type: 'int', name: 'sub' }] + ) + + assert_nothing_raised { validate!(schema, 'sub' => 1) } + + assert_failed_validation('at .sub expected type int, got null') do + validate!(schema, {}) + end + + assert_failed_validation('at . expected type record, got float with value 1.2') do + validate!(schema, 1.2) + end + + assert_failed_validation('at .sub expected type int, got float with value 1.2') do + validate!(schema, 'sub' => 1.2) + end + + assert_failed_validation('at .sub expected type int, got null') do + validate!(schema, {}) + end + end + + def test_validate_array + schema = hash_to_schema(type: 'array', + name: 'person', + items: [{ type: 'int', name: 'height' }]) + + assert_nothing_raised { validate!(schema, []) } + + assert_failed_validation 'at . expected type array, got null' do + validate!(schema, nil) + end + + assert_failed_validation('at .[0] expected type int, got null') do + validate!(schema, [nil]) + end + + assert_failed_validation('at .[3] expected type int, got string with value "so wrong"') do + validate!(schema, [1, 3, 9, 'so wrong']) + end + end + + def test_validate_enum + schema = hash_to_schema(type: 'enum', + name: 'person', + symbols: %w(one two three)) + + assert_nothing_raised { validate!(schema, 'one') } + + assert_failed_validation('at . expected enum with values ["one", "two", "three"], got string with value "five"') do + validate!(schema, 'five') + end + end + + def test_validate_union_on_primitive_types + schema = hash_to_schema( + name: 'should_not_matter', + type: 'record', + fields: [ + { name: 'what_ever', type: %w(long string) } + ] + ) + + assert_failed_validation('at .what_ever expected union of [\'long\', \'string\'], got null') { + validate!(schema, 'what_ever' => nil) + } + end + + def test_validate_union_of_nil_and_record_inside_array + schema = hash_to_schema( + name: 'this does not matter', + type: 'record', + fields: [ + { + name: 'person', + type: { + name: 'person_entry', + type: 'record', + fields: [ + { + name: 'houses', + type: [ + 'null', + { + name: 'houses_entry', + type: 'array', + items: [ + { + name: 'house_entry', + type: 'record', + fields: [ + { name: 'number_of_rooms', type: 'long' } + ] + } + ] + } + ], + } + ] + } + } + ] + ) + + assert_failed_validation('at .person expected type record, got null') { + validate!(schema, 'not at all' => nil) + } + assert_nothing_raised { validate!(schema, 'person' => {}) } + assert_nothing_raised { validate!(schema, 'person' => { houses: [] }) } + assert_nothing_raised { validate!(schema, 'person' => { 'houses' => [{ 'number_of_rooms' => 1 }] }) } + + message = 'at .person.houses[1].number_of_rooms expected type long, got string with value "not valid at all"' + assert_failed_validation(message) do + validate!( + schema, + 'person' => { + 'houses' => [ + { 'number_of_rooms' => 2 }, + { 'number_of_rooms' => 'not valid at all' } + ] + } + ) + end + end + + def test_validate_map + schema = hash_to_schema(type: 'map', + name: 'numbers', + values: [ + { name: 'some', type: 'int' } + ]) + + assert_nothing_raised { validate!(schema, 'some' => 1) } + + assert_failed_validation('at .some expected type int, got string with value "nope"') do + validate!(schema, 'some' => 'nope') + end + + assert_failed_validation("at . unexpected key type 'Symbol' in map") do + validate!(schema, some: 1) + end + end + + def test_validate_deep_record + schema = hash_to_schema(type: 'record', + name: 'person', + fields: [ + { + name: 'head', + type: { + name: 'head', + type: 'record', + fields: [ + { + name: 'hair', + type: { + name: 'hair', + type: 'record', + fields: [ + { + name: 'color', + type: 'string' + } + ] + } + } + ] + } + } + ]) + + assert_nothing_raised { validate!(schema, 'head' => { 'hair' => { 'color' => 'black' } }) } + + assert_failed_validation('at .head.hair.color expected type string, got null') do + validate!(schema, 'head' => { 'hair' => { 'color' => nil } }) + end + + assert_failed_validation('at .head.hair.color expected type string, got null') do + validate!(schema, 'head' => { 'hair' => {} }) + end + + assert_failed_validation('at .head.hair expected type record, got null') do + validate!(schema, 'head' => {}) + end + + assert_failed_validation('at . expected type record, got null') do + validate!(schema, nil) + end + end + + def test_validate_deep_record_with_array + schema = hash_to_schema(type: 'record', + name: 'fruits', + fields: [ + { + name: 'fruits', + type: { + name: 'fruits', + type: 'array', + items: [ + { + name: 'fruit', + type: 'record', + fields: [ + { name: 'name', type: 'string' }, + { name: 'weight', type: 'float' } + ] + } + ] + } + } + ]) + assert_nothing_raised { validate!(schema, 'fruits' => [{ 'name' => 'apple', 'weight' => 30.2 }]) } + + assert_failed_validation('at .fruits[0].name expected type string, got null') do + validate!(schema, 'fruits' => [{ 'name' => nil, 'weight' => 30.2 }]) + end + + assert_failed_validation('at .fruits expected type array, got int with value 1') do + validate!(schema, 'fruits' => 1) + end + end + + def test_validate_multiple_errors + schema = hash_to_schema(type: 'array', + name: 'ages', + items: [ + { type: 'int', name: 'age' } + ]) + + exception = assert_raise(Avro::SchemaValidator::ValidationError) do + validate!(schema, [nil, 'e']) + end + assert_equal 2, exception.result.errors.size + assert_equal( + "at .[0] expected type int, got null\nat .[1] expected type int, got string with value \"e\"", + exception.to_s + ) + end +end From 874fff780dfe6d6ee8a3fa13f461110794b56f35 Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Thu, 6 Apr 2017 13:39:48 -0400 Subject: [PATCH 16/22] [add] resolution code using new reusable visitor. --- .../avro/compiler/idl/ResolvingVisitor.java | 166 ++++++++++++++ .../avro/compiler/idl/SchemaResolver.java | 203 +++++------------- .../avro/compiler/schema/CloningVisitor.java | 155 +++++++++++++ lang/java/compiler/src/test/idl/cycle.avdl | 24 +++ .../avro/compiler/idl/SchemaResolverTest.java | 44 ++++ .../avro/compiler/schema/SchemasTest.java | 87 ++++++++ 6 files changed, 525 insertions(+), 154 deletions(-) create mode 100644 lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/ResolvingVisitor.java create mode 100644 lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/CloningVisitor.java create mode 100644 lang/java/compiler/src/test/idl/cycle.avdl create mode 100644 lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/SchemaResolverTest.java create mode 100644 lang/java/compiler/src/test/java/org/apache/avro/compiler/schema/SchemasTest.java diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/ResolvingVisitor.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/ResolvingVisitor.java new file mode 100644 index 00000000000..e2015985c72 --- /dev/null +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/ResolvingVisitor.java @@ -0,0 +1,166 @@ + +package org.apache.avro.compiler.idl; + +import avro.shaded.com.google.common.base.Function; +import java.util.ArrayList; +import java.util.IdentityHashMap; +import java.util.List; +import org.apache.avro.AvroTypeException; +import org.apache.avro.Schema; +import org.apache.avro.Schema.Field; +import org.apache.avro.compiler.schema.SchemaVisitor; +import org.apache.avro.compiler.schema.SchemaVisitorAction; +import org.apache.avro.compiler.schema.Schemas; + +/** + * this visitor will create a clone of the original Schema and will also resolve all unresolved schemas + * + * by default. what attributes are copied is customizable. + * @author zoly + */ +public final class ResolvingVisitor implements SchemaVisitor { + + private final IdentityHashMap replace; + private final Function symbolTable; + + private final Schema root; + + + public ResolvingVisitor(final Schema root, final IdentityHashMap replace, + final Function symbolTable) { + this.replace = replace; + this.symbolTable = symbolTable; + this.root = root; + } + + @Override + public SchemaVisitorAction visitTerminal(final Schema terminal) { + Schema.Type type = terminal.getType(); + Schema newSchema; + switch (type) { + case RECORD: // recursion. + case ARRAY: + case MAP: + case UNION: + if (!replace.containsKey(terminal)) { + throw new IllegalStateException("Schema " + terminal + " must be already processed"); + } + return SchemaVisitorAction.CONTINUE; + case BOOLEAN: + case BYTES: + case DOUBLE: + case FLOAT: + case INT: + case LONG: + case NULL: + case STRING: + newSchema = Schema.create(type); + break; + case ENUM: + newSchema = Schema.createEnum(terminal.getName(), terminal.getDoc(), + terminal.getNamespace(), terminal.getEnumSymbols()); + break; + case FIXED: + newSchema = Schema.createFixed(terminal.getName(), terminal.getDoc(), + terminal.getNamespace(), terminal.getFixedSize()); + break; + default: + throw new IllegalStateException("Unsupported schema " + terminal); + } + copyAllProperties(terminal, newSchema); + replace.put(terminal, newSchema); + return SchemaVisitorAction.CONTINUE; + } + + public static void copyAllProperties(final Schema first, final Schema second) { + Schemas.copyLogicalTypes(first, second); + Schemas.copyAliases(first, second); + Schemas.copyProperties(first, second); + } + + public static void copyAllProperties(final Field first, final Field second) { + Schemas.copyAliases(first, second); + Schemas.copyProperties(first, second); + } + + @Override + public SchemaVisitorAction visitNonTerminal(final Schema nt) { + Schema.Type type = nt.getType(); + if (type == Schema.Type.RECORD) { + if (SchemaResolver.isUnresolvedSchema(nt)) { + // unresolved schema will get a replacement that we already encountered, + // or we will attempt to resolve. + final String unresolvedSchemaName = SchemaResolver.getUnresolvedSchemaName(nt); + Schema resSchema = symbolTable.apply(unresolvedSchemaName); + if (resSchema == null) { + throw new AvroTypeException("Unable to resolve " + unresolvedSchemaName); + } + Schema replacement = replace.get(resSchema); + if (replacement == null) { + replace.put(nt, Schemas.visit(resSchema, new ResolvingVisitor(resSchema, + new IdentityHashMap(), symbolTable))); + } else { + replace.put(nt, replacement); + } + } else { + // create a fieldless clone. Fields will be added in afterVisitNonTerminal. + Schema newSchema = Schema.createRecord(nt.getName(), nt.getDoc(), nt.getNamespace(), nt.isError()); + copyAllProperties(nt, newSchema); + replace.put(nt, newSchema); + } + } + return SchemaVisitorAction.CONTINUE; + } + + @Override + public SchemaVisitorAction afterVisitNonTerminal(final Schema nt) { + Schema.Type type = nt.getType(); + Schema newSchema; + switch (type) { + case RECORD: + if (!SchemaResolver.isUnresolvedSchema(nt)) { + newSchema = replace.get(nt); + List fields = nt.getFields(); + List newFields = new ArrayList(fields.size()); + for (Schema.Field field : fields) { + Schema.Field newField = new Schema.Field(field.name(), replace.get(field.schema()), + field.doc(), field.defaultVal(), field.order()); + copyAllProperties(field, newField); + newFields.add(newField); + } + newSchema.setFields(newFields); + } + return SchemaVisitorAction.CONTINUE; + case UNION: + List types = nt.getTypes(); + List newTypes = new ArrayList(types.size()); + for (Schema sch : types) { + newTypes.add(replace.get(sch)); + } + newSchema = Schema.createUnion(newTypes); + break; + case ARRAY: + newSchema = Schema.createArray(replace.get(nt.getElementType())); + break; + case MAP: + newSchema = Schema.createMap(replace.get(nt.getValueType())); + break; + default: + throw new IllegalStateException("Illegal type " + type + ", schema " + nt); + } + copyAllProperties(nt, newSchema); + replace.put(nt, newSchema); + return SchemaVisitorAction.CONTINUE; + } + + @Override + public Schema get() { + return replace.get(root); + } + + @Override + public String toString() { + return "ResolvingVisitor{" + "replace=" + replace + ", symbolTable=" + symbolTable + ", root=" + root + '}'; + } + +} diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java index 6aed7878283..f456a181af5 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java @@ -15,10 +15,11 @@ */ package org.apache.avro.compiler.idl; +import avro.shaded.com.google.common.base.Function; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import org.apache.avro.Protocol; @@ -28,7 +29,6 @@ /** * Utility class to resolve schemas that are unavailable at the time they are referenced in the IDL. */ - final class SchemaResolver { private SchemaResolver() { @@ -40,21 +40,36 @@ private SchemaResolver() { private static final String UR_SCHEMA_NS = "org.apache.avro.compiler"; + /** + * Create a schema to represent a "unresolved" schema. + * (used to represent a schema where the definition is not known at the time) + * This concept might be generalizable... + * @param name + * @return + */ static Schema unresolvedSchema(final String name) { - - Schema schema = Schema.createRecord(UR_SCHEMA_NAME, "unresolved schema", UR_SCHEMA_NS, false, Collections.EMPTY_LIST); schema.addProp(UR_SCHEMA_ATTR, name); return schema; } + /** + * Is this a unresolved schema. + * @param schema + * @return + */ static boolean isUnresolvedSchema(final Schema schema) { return (schema.getType() == Schema.Type.RECORD && schema.getProp(UR_SCHEMA_ATTR) != null && UR_SCHEMA_NAME.equals(schema.getName()) && UR_SCHEMA_NS.equals(schema.getNamespace())); } + /** + * get the unresolved schema name. + * @param schema + * @return + */ static String getUnresolvedSchemaName(final Schema schema) { if (!isUnresolvedSchema(schema)) { throw new IllegalArgumentException("Not a unresolved schema: " + schema); @@ -68,34 +83,35 @@ static String getUnresolvedSchemaName(final Schema schema) { } /** - * Resolve all unresolved schema references from a protocol. - * @param protocol - the protocol with unresolved schema references. - * @return - a new protocol instance based on the provided protocol with all unresolved schema references resolved. + * Will clone the provided protocol while resolving all unreferenced schemas + * @param protocol + * @return */ static Protocol resolve(final Protocol protocol) { Protocol result = new Protocol(protocol.getName(), protocol.getDoc(), protocol.getNamespace()); final Collection types = protocol.getTypes(); + // replace unresolved schemas. List newSchemas = new ArrayList(types.size()); - Map resolved = new HashMap(); + IdentityHashMap replacements = new IdentityHashMap(); for (Schema schema : types) { - newSchemas.add(resolve(schema, protocol, resolved)); + newSchemas.add(Schemas.visit(schema, new ResolvingVisitor(schema, replacements, new SymbolTable(protocol)))); } result.setTypes(newSchemas); // replace types with resolved ones + // Resolve all schemas refferenced by protocol Messages. for (Map.Entry entry : protocol.getMessages().entrySet()) { Protocol.Message value = entry.getValue(); Protocol.Message nvalue; if (value.isOneWay()) { - Schema request = value.getRequest(); + Schema replacement = resolve(replacements, value.getRequest(), protocol); nvalue = result.createMessage(value.getName(), value.getDoc(), - value.getObjectProps(), getResolvedSchema(request, resolved)); + value.getObjectProps(), replacement); } else { - Schema request = value.getRequest(); - Schema response = value.getResponse(); - Schema errors = value.getErrors(); + Schema request = resolve(replacements, value.getRequest(), protocol); + Schema response = resolve(replacements, value.getResponse(), protocol); + Schema errors = resolve(replacements, value.getErrors(), protocol); nvalue = result.createMessage(value.getName(), value.getDoc(), - value.getObjectProps(), getResolvedSchema(request, resolved), - getResolvedSchema(response, resolved), getResolvedSchema(errors, resolved)); + value.getObjectProps(), request, response, errors); } result.getMessages().put(entry.getKey(), nvalue); } @@ -103,148 +119,27 @@ static Protocol resolve(final Protocol protocol) { return result; } - - /** - * Resolve all unresolved schema references. - * @param schema - the schema to resolved references for. - * @param protocol - the protocol we resolve the schema's for. - * (we lookup all unresolved schema references in the protocol) - * @param resolved - a map of all resolved schema's so far. - * @return - a instance of the resolved schema. - */ - static Schema resolve(final Schema schema, final Protocol protocol, final Map resolved) { - final String fullName = schema.getFullName(); - if (fullName != null && resolved.containsKey(fullName)) { - return resolved.get(schema.getFullName()); - } else if (isUnresolvedSchema(schema)) { - final String unresolvedSchemaName = getUnresolvedSchemaName(schema); - Schema type = protocol.getType(unresolvedSchemaName); - if (type == null) { - throw new IllegalArgumentException("Cannot resolve " + unresolvedSchemaName); - } - return resolve(type, protocol, resolved); - } else { - switch (schema.getType()) { - case RECORD: - Schema createRecord = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), - schema.isError()); - resolved.put(schema.getFullName(), createRecord); - final List currFields = schema.getFields(); - List newFields = new ArrayList(currFields.size()); - for (Schema.Field field : currFields) { - if (field.name().equals("hash")) { - System.err.println(field); - } - Schema.Field nf = new Schema.Field(field.name(), resolve(field.schema(), protocol, resolved), - field.doc(), field.defaultVal(), field.order()); - Schemas.copyAliases(field, nf); - Schemas.copyProperties(field, nf); - newFields.add(nf); - } - createRecord.setFields(newFields); - Schemas.copyLogicalTypes(schema, createRecord); - Schemas.copyProperties(schema, createRecord); - return createRecord; - case MAP: - Schema result = Schema.createMap(resolve(schema.getValueType(), protocol, resolved)); - Schemas.copyProperties(schema, result); - return result; - case ARRAY: - Schema aresult = Schema.createArray(resolve(schema.getElementType(), protocol, resolved)); - Schemas.copyProperties(schema, aresult); - return aresult; - case UNION: - final List uTypes = schema.getTypes(); - List newTypes = new ArrayList(uTypes.size()); - for (Schema s : uTypes) { - newTypes.add(resolve(s, protocol, resolved)); - } - Schema bresult = Schema.createUnion(newTypes); - Schemas.copyProperties(schema, bresult); - return bresult; - case ENUM: - case FIXED: - case STRING: - case BYTES: - case INT: - case LONG: - case FLOAT: - case DOUBLE: - case BOOLEAN: - case NULL: - return schema; - default: - throw new RuntimeException("Unknown type: " + schema); - } + private static Schema resolve(final IdentityHashMap replacements, + final Schema request, final Protocol protocol) { + Schema replacement = replacements.get(request); + if (replacement == null) { + replacement = Schemas.visit(request, new ResolvingVisitor(request, replacements, + new SymbolTable(protocol))); } + return replacement; } - /** - * get the resolved schema. - * @param schema - the schema we want to get the resolved equivalent for. - * @param resolved - a Map wil all resolved schemas - * @return - the resolved schema. - */ - public static Schema getResolvedSchema(final Schema schema, final Map resolved) { - if (schema == null) { - return null; + private static class SymbolTable implements Function { + + private final Protocol symbolTable; + + public SymbolTable(Protocol symbolTable) { + this.symbolTable = symbolTable; } - final String fullName = schema.getFullName(); - if (fullName != null && resolved.containsKey(fullName)) { - return resolved.get(schema.getFullName()); - } else { - switch (schema.getType()) { - case RECORD: - Schema createRecord = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), - schema.isError()); - resolved.put(schema.getFullName(), createRecord); - final List currFields = schema.getFields(); - List newFields = new ArrayList(currFields.size()); - for (Schema.Field field : currFields) { - if (field.name().equals("hash")) { - System.err.println(field); - } - Schema.Field nf = new Schema.Field(field.name(), getResolvedSchema(field.schema(), resolved), - field.doc(), field.defaultVal(), field.order()); - Schemas.copyAliases(field, nf); - Schemas.copyProperties(field, nf); - newFields.add(nf); - } - createRecord.setFields(newFields); - Schemas.copyLogicalTypes(schema, createRecord); - Schemas.copyProperties(schema, createRecord); - return createRecord; - case MAP: - Schema createMap = Schema.createMap(getResolvedSchema(schema.getValueType(), resolved)); - Schemas.copyProperties(schema, createMap); - return createMap; - case ARRAY: - Schema createArray = Schema.createArray(getResolvedSchema(schema.getElementType(), resolved)); - Schemas.copyProperties(schema, createArray); - return createArray; - case UNION: - final List uTypes = schema.getTypes(); - List newTypes = new ArrayList(uTypes.size()); - for (Schema s : uTypes) { - newTypes.add(getResolvedSchema(s, resolved)); - } - Schema createUnion = Schema.createUnion(newTypes); - Schemas.copyProperties(schema, createUnion); - return createUnion; - case ENUM: - case FIXED: - case STRING: - case BYTES: - case INT: - case LONG: - case FLOAT: - case DOUBLE: - case BOOLEAN: - case NULL: - return schema; - default: - throw new RuntimeException("Unknown type: " + schema); - } + + @Override + public Schema apply(final String f) { + return symbolTable.getType(f); } } diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/CloningVisitor.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/CloningVisitor.java new file mode 100644 index 00000000000..8c6423bf9e0 --- /dev/null +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/CloningVisitor.java @@ -0,0 +1,155 @@ + +package org.apache.avro.compiler.schema; + +import java.util.ArrayList; +import java.util.IdentityHashMap; +import java.util.List; +import org.apache.avro.Schema; +import static org.apache.avro.Schema.Type.RECORD; + +/** + * this visitor will create a clone of the original Schema with docs and other nonesential fields stripped + * by default. what attributes are copied is customizable. + * @author zoly + */ +public final class CloningVisitor implements SchemaVisitor { + + private final IdentityHashMap replace = new IdentityHashMap(); + + private final Schema root; + + private final PropertyCopier copyProperties; + + private final boolean copyDocs; + + public interface PropertyCopier { + void copy(Schema first, Schema second); + void copy(Schema.Field first, Schema.Field second); + } + + /** + * copy only serialization necessary fields. + * @param root + */ + public CloningVisitor(final Schema root) { + this(new PropertyCopier() { + @Override + public void copy(final Schema first, final Schema second) { + Schemas.copyLogicalTypes(first, second); + Schemas.copyAliases(first, second); + } + + @Override + public void copy(final Schema.Field first, final Schema.Field second) { + Schemas.copyAliases(first, second); + } + }, false, root); + } + + public CloningVisitor(final PropertyCopier copyProperties, final boolean copyDocs, final Schema root) { + this.copyProperties = copyProperties; + this.copyDocs = copyDocs; + this.root = root; + } + + @Override + public SchemaVisitorAction visitTerminal(final Schema terminal) { + Schema.Type type = terminal.getType(); + Schema newSchema; + switch (type) { + case RECORD: // recursion. + case ARRAY: + case MAP: + case UNION: + if (!replace.containsKey(terminal)) { + throw new IllegalStateException("Schema " + terminal + " must be already processed"); + } + return SchemaVisitorAction.CONTINUE; + case BOOLEAN: + case BYTES: + case DOUBLE: + case FLOAT: + case INT: + case LONG: + case NULL: + case STRING: + newSchema = Schema.create(type); + break; + case ENUM: + newSchema = Schema.createEnum(terminal.getName(), copyDocs ? terminal.getDoc() : null, + terminal.getNamespace(), terminal.getEnumSymbols()); + break; + case FIXED: + newSchema = Schema.createFixed(terminal.getName(), copyDocs ? terminal.getDoc() : null, + terminal.getNamespace(), terminal.getFixedSize()); + break; + default: + throw new IllegalStateException("Unsupported schema " + terminal); + } + copyProperties.copy(terminal, newSchema); + replace.put(terminal, newSchema); + return SchemaVisitorAction.CONTINUE; + } + + @Override + public SchemaVisitorAction visitNonTerminal(final Schema nt) { + Schema.Type type = nt.getType(); + if (type == RECORD) { + Schema newSchema = Schema.createRecord(nt.getName(), copyDocs ? nt.getDoc() : null, + nt.getNamespace(), nt.isError()); + copyProperties.copy(nt, newSchema); + replace.put(nt, newSchema); + } + return SchemaVisitorAction.CONTINUE; + } + + @Override + public SchemaVisitorAction afterVisitNonTerminal(final Schema nt) { + Schema.Type type = nt.getType(); + Schema newSchema; + switch (type) { + case RECORD: + newSchema = replace.get(nt); + List fields = nt.getFields(); + List newFields = new ArrayList(fields.size()); + for (Schema.Field field : fields) { + Schema.Field newField = new Schema.Field(field.name(), replace.get(field.schema()), + copyDocs ? field.doc() : null, field.defaultVal(), field.order()); + copyProperties.copy(field, newField); + newFields.add(newField); + } + newSchema.setFields(newFields); + return SchemaVisitorAction.CONTINUE; + case UNION: + List types = nt.getTypes(); + List newTypes = new ArrayList(types.size()); + for (Schema sch : types) { + newTypes.add(replace.get(sch)); + } + newSchema = Schema.createUnion(newTypes); + break; + case ARRAY: + newSchema = Schema.createArray(replace.get(nt.getElementType())); + break; + case MAP: + newSchema = Schema.createMap(replace.get(nt.getValueType())); + break; + default: + throw new IllegalStateException("Illegal type " + type + ", schema " + nt); + } + copyProperties.copy(nt, newSchema); + replace.put(nt, newSchema); + return SchemaVisitorAction.CONTINUE; + } + + @Override + public Schema get() { + return replace.get(root); + } + + @Override + public String toString() { + return "CloningVisitor{" + "replace=" + replace + ", root=" + root + '}'; + } + +} diff --git a/lang/java/compiler/src/test/idl/cycle.avdl b/lang/java/compiler/src/test/idl/cycle.avdl new file mode 100644 index 00000000000..78b0fc1b7ed --- /dev/null +++ b/lang/java/compiler/src/test/idl/cycle.avdl @@ -0,0 +1,24 @@ +@namespace("org.apache.avro.gen.test") +protocol Cycle { + + record Record1 { + string fString = ""; + Record3 rec3; + } + + record Record2 { + TestFixed fFixed; + int val; + union {null, Record1} fRec1; + } + + record Record3 { + TestEnum fEnum; + Record2 rec2; + } + + enum TestEnum { bla, blu } + + fixed TestFixed(16); + +} diff --git a/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/SchemaResolverTest.java b/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/SchemaResolverTest.java new file mode 100644 index 00000000000..61212f7e7e4 --- /dev/null +++ b/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/SchemaResolverTest.java @@ -0,0 +1,44 @@ +/* + * Copyright 2017 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.compiler.idl; + +import java.io.File; +import java.io.IOException; +import java.net.MalformedURLException; +import org.apache.avro.Protocol; +import org.junit.Assert; +import org.junit.Test; + +/** + * + * @author zoly + */ +public class SchemaResolverTest { + + + @Test + public void testResolving() throws ParseException, MalformedURLException, IOException { + File file = new File("."); + String currentWorkPath = file.getAbsolutePath(); + String testIdl = currentWorkPath + File.separator + "src" + File.separator + "test" + + File.separator + "idl" + File.separator + "cycle.avdl"; + Idl compiler = new Idl(new File(testIdl)); + Protocol protocol = compiler.CompilationUnit(); + System.out.println(protocol); + Assert.assertEquals(5, protocol.getTypes().size()); + } + +} diff --git a/lang/java/compiler/src/test/java/org/apache/avro/compiler/schema/SchemasTest.java b/lang/java/compiler/src/test/java/org/apache/avro/compiler/schema/SchemasTest.java new file mode 100644 index 00000000000..545901e53ca --- /dev/null +++ b/lang/java/compiler/src/test/java/org/apache/avro/compiler/schema/SchemasTest.java @@ -0,0 +1,87 @@ +/* + * Copyright 2017 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.compiler.schema; + +import org.apache.avro.Schema; +import org.apache.avro.SchemaCompatibility; +import org.junit.Assert; +import org.junit.Test; + +public class SchemasTest { + + private static final String SCHEMA = "{\"type\":\"record\",\"name\":\"SampleNode\",\"doc\":\"caca\"," + + "\"namespace\":\"org.spf4j.ssdump2.avro\",\n" + + " \"fields\":[\n" + + " {\"name\":\"count\",\"type\":\"int\",\"default\":0,\"doc\":\"caca\"},\n" + + " {\"name\":\"subNodes\",\"type\":\n" + + " {\"type\":\"array\",\"items\":{\n" + + " \"type\":\"record\",\"name\":\"SamplePair\",\n" + + " \"fields\":[\n" + + " {\"name\":\"method\",\"type\":\n" + + " {\"type\":\"record\",\"name\":\"Method\",\n" + + " \"fields\":[\n" + + " {\"name\":\"declaringClass\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},\n" + + " {\"name\":\"methodName\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}}\n" + + " ]}},\n" + + " {\"name\":\"node\",\"type\":\"SampleNode\"}]}}}]}"; + + private static class PrintingVisitor implements SchemaVisitor { + + + @Override + public SchemaVisitorAction visitTerminal(Schema terminal) { + System.out.println("Terminal: " + terminal.getFullName()); + return SchemaVisitorAction.CONTINUE; + } + + @Override + public SchemaVisitorAction visitNonTerminal(Schema terminal) { + System.out.println("NONTerminal start: " + terminal.getFullName()); + return SchemaVisitorAction.CONTINUE; + } + + @Override + public SchemaVisitorAction afterVisitNonTerminal(Schema terminal) { + System.out.println("NONTerminal end: " + terminal.getFullName()); + return SchemaVisitorAction.CONTINUE; + } + + @Override + public Object get() { + return null; + } + } + + + + @Test + public void textCloning() { + Schema recSchema = new Schema.Parser().parse(SCHEMA); + Schemas.visit(recSchema, new PrintingVisitor()); + + + Schema trimmed = Schemas.visit(recSchema, new CloningVisitor(recSchema)); + Assert.assertNull(trimmed.getDoc()); + Assert.assertNotNull(recSchema.getDoc()); + + SchemaCompatibility.SchemaCompatibilityType compat = + SchemaCompatibility.checkReaderWriterCompatibility(trimmed, recSchema).getType(); + Assert.assertEquals(SchemaCompatibility.SchemaCompatibilityType.COMPATIBLE, compat); + compat = SchemaCompatibility.checkReaderWriterCompatibility(recSchema, trimmed).getType(); + Assert.assertEquals(SchemaCompatibility.SchemaCompatibilityType.COMPATIBLE, compat); + } + +} From ce1654df5e6b3c9e4b142678b61d36f19c9da359 Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Fri, 28 Apr 2017 13:56:17 -0400 Subject: [PATCH 17/22] [fix] eliminate FB annotations to reduce the scope of this change. --- lang/java/compiler/pom.xml | 11 ++++++----- .../apache/avro/compiler/schema/SchemaVisitor.java | 10 ---------- .../java/org/apache/avro/compiler/schema/Schemas.java | 2 -- 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/lang/java/compiler/pom.xml b/lang/java/compiler/pom.xml index 2a35c456bbf..4b0021515ba 100644 --- a/lang/java/compiler/pom.xml +++ b/lang/java/compiler/pom.xml @@ -138,11 +138,12 @@ guava ${guava.version} test - - - com.google.code.findbugs - jsr305 - 3.0.1 + + + com.google.code.findbugs + jsr305 + + diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/SchemaVisitor.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/SchemaVisitor.java index df667f46b66..41983555d72 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/SchemaVisitor.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/SchemaVisitor.java @@ -17,15 +17,11 @@ */ package org.apache.avro.compiler.schema; -import javax.annotation.CheckReturnValue; -import javax.annotation.Nonnull; -import javax.annotation.ParametersAreNonnullByDefault; import org.apache.avro.Schema; /** * @author zoly */ -@ParametersAreNonnullByDefault public interface SchemaVisitor { /** @@ -35,8 +31,6 @@ public interface SchemaVisitor { * @param terminal * @return */ - @Nonnull - @CheckReturnValue SchemaVisitorAction visitTerminal(Schema terminal); /** @@ -44,8 +38,6 @@ public interface SchemaVisitor { * @param nonTerminal * @return */ - @Nonnull - @CheckReturnValue SchemaVisitorAction visitNonTerminal(Schema nonTerminal); /** @@ -53,8 +45,6 @@ public interface SchemaVisitor { * @param nonTerminal * @return */ - @Nonnull - @CheckReturnValue SchemaVisitorAction afterVisitNonTerminal(Schema nonTerminal); diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/Schemas.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/Schemas.java index 82cf0bbe943..0196a7178c2 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/Schemas.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/Schemas.java @@ -10,7 +10,6 @@ import java.util.Iterator; import java.util.Map; import java.util.Set; -import javax.annotation.ParametersAreNonnullByDefault; import org.apache.avro.JsonProperties; import org.apache.avro.LogicalType; import org.apache.avro.Schema; @@ -22,7 +21,6 @@ * * @author zoly */ -@ParametersAreNonnullByDefault public final class Schemas { private Schemas() { From ce0bad58393fb1d1d5a446062aca16f1a68d1260 Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Sat, 29 Apr 2017 09:24:09 -0400 Subject: [PATCH 18/22] [fix] fix indentation --- lang/java/compiler/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lang/java/compiler/pom.xml b/lang/java/compiler/pom.xml index 4b0021515ba..3ff010197f7 100644 --- a/lang/java/compiler/pom.xml +++ b/lang/java/compiler/pom.xml @@ -140,9 +140,9 @@ test - com.google.code.findbugs - jsr305 - + com.google.code.findbugs + jsr305 + From 9132015450a2ad6f56cd582b393e8f1b8df573c9 Mon Sep 17 00:00:00 2001 From: Thiruvalluvan M G Date: Sun, 30 Apr 2017 21:02:02 +0530 Subject: [PATCH 19/22] Added more tests and fixed a couple of bugs. Also formatted the code --- .../avro/compiler/idl/ResolvingVisitor.java | 1 - .../avro/compiler/idl/SchemaResolver.java | 27 +- .../avro/compiler/schema/CloningVisitor.java | 87 ++--- .../avro/compiler/schema/SchemaVisitor.java | 41 ++- .../compiler/schema/SchemaVisitorAction.java | 71 ++-- .../apache/avro/compiler/schema/Schemas.java | 47 +-- .../avro/compiler/idl/SchemaResolverTest.java | 44 --- .../apache/avro/compiler/idl/TestCycle.java | 6 +- .../avro/compiler/idl/TestSchemaResolver.java | 72 ++++ .../avro/compiler/schema/SchemasTest.java | 87 ----- .../avro/compiler/schema/TestSchemas.java | 347 ++++++++++++++++++ 11 files changed, 552 insertions(+), 278 deletions(-) delete mode 100644 lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/SchemaResolverTest.java create mode 100644 lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestSchemaResolver.java delete mode 100644 lang/java/compiler/src/test/java/org/apache/avro/compiler/schema/SchemasTest.java create mode 100644 lang/java/compiler/src/test/java/org/apache/avro/compiler/schema/TestSchemas.java diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/ResolvingVisitor.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/ResolvingVisitor.java index e2015985c72..35fb44b30a0 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/ResolvingVisitor.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/ResolvingVisitor.java @@ -16,7 +16,6 @@ * this visitor will create a clone of the original Schema and will also resolve all unresolved schemas * * by default. what attributes are copied is customizable. - * @author zoly */ public final class ResolvingVisitor implements SchemaVisitor { diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java index f456a181af5..596523ca0b0 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java @@ -16,12 +16,14 @@ package org.apache.avro.compiler.idl; import avro.shaded.com.google.common.base.Function; + import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.IdentityHashMap; import java.util.List; import java.util.Map; + import org.apache.avro.Protocol; import org.apache.avro.Schema; import org.apache.avro.compiler.schema.Schemas; @@ -44,29 +46,32 @@ private SchemaResolver() { * Create a schema to represent a "unresolved" schema. * (used to represent a schema where the definition is not known at the time) * This concept might be generalizable... + * * @param name * @return */ static Schema unresolvedSchema(final String name) { Schema schema = Schema.createRecord(UR_SCHEMA_NAME, "unresolved schema", - UR_SCHEMA_NS, false, Collections.EMPTY_LIST); + UR_SCHEMA_NS, false, Collections.EMPTY_LIST); schema.addProp(UR_SCHEMA_ATTR, name); return schema; } /** * Is this a unresolved schema. + * * @param schema * @return */ static boolean isUnresolvedSchema(final Schema schema) { return (schema.getType() == Schema.Type.RECORD && schema.getProp(UR_SCHEMA_ATTR) != null - && UR_SCHEMA_NAME.equals(schema.getName()) - && UR_SCHEMA_NS.equals(schema.getNamespace())); + && UR_SCHEMA_NAME.equals(schema.getName()) + && UR_SCHEMA_NS.equals(schema.getNamespace())); } /** * get the unresolved schema name. + * * @param schema * @return */ @@ -74,16 +79,12 @@ static String getUnresolvedSchemaName(final Schema schema) { if (!isUnresolvedSchema(schema)) { throw new IllegalArgumentException("Not a unresolved schema: " + schema); } - String name = schema.getProp(UR_SCHEMA_ATTR); - if (name == null) { - throw new IllegalArgumentException("Schema " + schema + " must have attribute: " + UR_SCHEMA_ATTR); - } else { - return name; - } + return schema.getProp(UR_SCHEMA_ATTR); } /** * Will clone the provided protocol while resolving all unreferenced schemas + * * @param protocol * @return */ @@ -105,13 +106,13 @@ static Protocol resolve(final Protocol protocol) { if (value.isOneWay()) { Schema replacement = resolve(replacements, value.getRequest(), protocol); nvalue = result.createMessage(value.getName(), value.getDoc(), - value.getObjectProps(), replacement); + value.getObjectProps(), replacement); } else { Schema request = resolve(replacements, value.getRequest(), protocol); Schema response = resolve(replacements, value.getResponse(), protocol); Schema errors = resolve(replacements, value.getErrors(), protocol); nvalue = result.createMessage(value.getName(), value.getDoc(), - value.getObjectProps(), request, response, errors); + value.getObjectProps(), request, response, errors); } result.getMessages().put(entry.getKey(), nvalue); } @@ -120,11 +121,11 @@ static Protocol resolve(final Protocol protocol) { } private static Schema resolve(final IdentityHashMap replacements, - final Schema request, final Protocol protocol) { + final Schema request, final Protocol protocol) { Schema replacement = replacements.get(request); if (replacement == null) { replacement = Schemas.visit(request, new ResolvingVisitor(request, replacements, - new SymbolTable(protocol))); + new SymbolTable(protocol))); } return replacement; } diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/CloningVisitor.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/CloningVisitor.java index 8c6423bf9e0..ccc222a0281 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/CloningVisitor.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/CloningVisitor.java @@ -4,13 +4,14 @@ import java.util.ArrayList; import java.util.IdentityHashMap; import java.util.List; + import org.apache.avro.Schema; + import static org.apache.avro.Schema.Type.RECORD; /** * this visitor will create a clone of the original Schema with docs and other nonesential fields stripped * by default. what attributes are copied is customizable. - * @author zoly */ public final class CloningVisitor implements SchemaVisitor { @@ -24,11 +25,13 @@ public final class CloningVisitor implements SchemaVisitor { public interface PropertyCopier { void copy(Schema first, Schema second); + void copy(Schema.Field first, Schema.Field second); } /** * copy only serialization necessary fields. + * * @param root */ public CloningVisitor(final Schema root) { @@ -41,7 +44,7 @@ public void copy(final Schema first, final Schema second) { @Override public void copy(final Schema.Field first, final Schema.Field second) { - Schemas.copyAliases(first, second); + Schemas.copyAliases(first, second); } }, false, root); } @@ -74,14 +77,14 @@ public SchemaVisitorAction visitTerminal(final Schema terminal) { case NULL: case STRING: newSchema = Schema.create(type); - break; + break; case ENUM: newSchema = Schema.createEnum(terminal.getName(), copyDocs ? terminal.getDoc() : null, - terminal.getNamespace(), terminal.getEnumSymbols()); + terminal.getNamespace(), terminal.getEnumSymbols()); break; case FIXED: newSchema = Schema.createFixed(terminal.getName(), copyDocs ? terminal.getDoc() : null, - terminal.getNamespace(), terminal.getFixedSize()); + terminal.getNamespace(), terminal.getFixedSize()); break; default: throw new IllegalStateException("Unsupported schema " + terminal); @@ -94,52 +97,52 @@ public SchemaVisitorAction visitTerminal(final Schema terminal) { @Override public SchemaVisitorAction visitNonTerminal(final Schema nt) { Schema.Type type = nt.getType(); - if (type == RECORD) { - Schema newSchema = Schema.createRecord(nt.getName(), copyDocs ? nt.getDoc() : null, - nt.getNamespace(), nt.isError()); - copyProperties.copy(nt, newSchema); - replace.put(nt, newSchema); + if (type == RECORD) { + Schema newSchema = Schema.createRecord(nt.getName(), copyDocs ? nt.getDoc() : null, + nt.getNamespace(), nt.isError()); + copyProperties.copy(nt, newSchema); + replace.put(nt, newSchema); } return SchemaVisitorAction.CONTINUE; } @Override public SchemaVisitorAction afterVisitNonTerminal(final Schema nt) { - Schema.Type type = nt.getType(); - Schema newSchema; - switch (type) { - case RECORD: - newSchema = replace.get(nt); - List fields = nt.getFields(); - List newFields = new ArrayList(fields.size()); - for (Schema.Field field : fields) { + Schema.Type type = nt.getType(); + Schema newSchema; + switch (type) { + case RECORD: + newSchema = replace.get(nt); + List fields = nt.getFields(); + List newFields = new ArrayList(fields.size()); + for (Schema.Field field : fields) { Schema.Field newField = new Schema.Field(field.name(), replace.get(field.schema()), - copyDocs ? field.doc() : null, field.defaultVal(), field.order()); + copyDocs ? field.doc() : null, field.defaultVal(), field.order()); copyProperties.copy(field, newField); newFields.add(newField); - } - newSchema.setFields(newFields); - return SchemaVisitorAction.CONTINUE; - case UNION: - List types = nt.getTypes(); - List newTypes = new ArrayList(types.size()); - for (Schema sch : types) { - newTypes.add(replace.get(sch)); - } - newSchema = Schema.createUnion(newTypes); - break; - case ARRAY: - newSchema = Schema.createArray(replace.get(nt.getElementType())); - break; - case MAP: - newSchema = Schema.createMap(replace.get(nt.getValueType())); - break; - default: - throw new IllegalStateException("Illegal type " + type + ", schema " + nt); - } - copyProperties.copy(nt, newSchema); - replace.put(nt, newSchema); - return SchemaVisitorAction.CONTINUE; + } + newSchema.setFields(newFields); + return SchemaVisitorAction.CONTINUE; + case UNION: + List types = nt.getTypes(); + List newTypes = new ArrayList(types.size()); + for (Schema sch : types) { + newTypes.add(replace.get(sch)); + } + newSchema = Schema.createUnion(newTypes); + break; + case ARRAY: + newSchema = Schema.createArray(replace.get(nt.getElementType())); + break; + case MAP: + newSchema = Schema.createMap(replace.get(nt.getValueType())); + break; + default: + throw new IllegalStateException("Illegal type " + type + ", schema " + nt); + } + copyProperties.copy(nt, newSchema); + replace.put(nt, newSchema); + return SchemaVisitorAction.CONTINUE; } @Override diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/SchemaVisitor.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/SchemaVisitor.java index 41983555d72..8000314f83c 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/SchemaVisitor.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/SchemaVisitor.java @@ -1,33 +1,31 @@ - /* - * Copyright (c) 2001 - 2016, Zoltan Farkas All Rights Reserved. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - */ +/* +* Copyright (c) 2001 - 2016, Zoltan Farkas All Rights Reserved. +* +* This library is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* This library is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with this program; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +*/ package org.apache.avro.compiler.schema; import org.apache.avro.Schema; -/** - * @author zoly - */ public interface SchemaVisitor { /** * Invoked for schemas that do not have "child" schemas (like string, int ...) * or for a previously encountered schema with children, * which will be treated as a terminal. (to avoid circular recursion) + * * @param terminal * @return */ @@ -35,6 +33,7 @@ public interface SchemaVisitor { /** * Invoked for schema with children before proceeding to visit the children. + * * @param nonTerminal * @return */ @@ -42,6 +41,7 @@ public interface SchemaVisitor { /** * Invoked for schemas with children after its children have been visited. + * * @param nonTerminal * @return */ @@ -50,6 +50,7 @@ public interface SchemaVisitor { /** * Invoked when visiting is complete. + * * @return a value which will be returned by the visit method. */ T get(); diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/SchemaVisitorAction.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/SchemaVisitorAction.java index 81157aad0a3..305ffbd591b 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/SchemaVisitorAction.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/SchemaVisitorAction.java @@ -1,44 +1,41 @@ - /* - * Copyright (c) 2001 - 2016, Zoltan Farkas All Rights Reserved. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - */ +/* +* Copyright (c) 2001 - 2016, Zoltan Farkas All Rights Reserved. +* +* This library is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* This library is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with this program; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +*/ package org.apache.avro.compiler.schema; -/** - * @author zoly - */ public enum SchemaVisitorAction { - /** - * continue visit. - */ - CONTINUE, - /** - * terminate visit. - */ - TERMINATE, - /** - * when returned from pre non terminal visit method the children of the non terminal are skipped. - * afterVisitNonTerminal for the current schema will not be invoked. - */ - SKIP_SUBTREE, - /** - * Skip visiting the siblings of this schema. - */ - SKIP_SIBLINGS; + /** + * continue visit. + */ + CONTINUE, + /** + * terminate visit. + */ + TERMINATE, + /** + * when returned from pre non terminal visit method the children of the non terminal are skipped. + * afterVisitNonTerminal for the current schema will not be invoked. + */ + SKIP_SUBTREE, + /** + * Skip visiting the siblings of this schema. + */ + SKIP_SIBLINGS; } diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/Schemas.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/Schemas.java index 0196a7178c2..ac39d883ecb 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/Schemas.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/Schemas.java @@ -3,6 +3,7 @@ import avro.shaded.com.google.common.base.Function; import avro.shaded.com.google.common.base.Supplier; import avro.shaded.com.google.common.collect.Lists; + import java.util.ArrayDeque; import java.util.Arrays; import java.util.Deque; @@ -10,6 +11,7 @@ import java.util.Iterator; import java.util.Map; import java.util.Set; + import org.apache.avro.JsonProperties; import org.apache.avro.LogicalType; import org.apache.avro.Schema; @@ -18,8 +20,6 @@ /** * Avro Schema utilities, to traverse... - * - * @author zoly */ public final class Schemas { @@ -73,7 +73,7 @@ public static boolean hasGeneratedJavaClass(final Schema schema) { public static String getJavaClassName(final Schema schema) { String namespace = schema.getNamespace(); - if (namespace == null || namespace.isEmpty()) { + if (namespace == null) { return SpecificCompiler.mangle(schema.getName()); } else { return namespace + '.' + SpecificCompiler.mangle(schema.getName()); @@ -105,12 +105,9 @@ public static T visit(final Schema start, final SchemaVisitor visitor) { case SKIP_SUBTREE: throw new UnsupportedOperationException(); case SKIP_SIBLINGS: - //CHECKSTYLE:OFF InnerAssignment - while ((current = dq.getLast()) instanceof Schema) { - // just skip + while (dq.getLast() instanceof Schema) { + dq.removeLast(); } - //CHECKSTYLE:ON - dq.addLast(current); break; case TERMINATE: return visitor.get(); @@ -129,12 +126,12 @@ public static T visit(final Schema start, final SchemaVisitor visitor) { break; case RECORD: terminate = visitNonTerminal(visitor, schema, dq, - Lists.transform(Lists.reverse(schema.getFields()), new Function() { - @Override - public Schema apply(Field f) { - return f.schema(); - } - })); + Lists.transform(Lists.reverse(schema.getFields()), new Function() { + @Override + public Schema apply(Field f) { + return f.schema(); + } + })); visited.put(schema, schema); break; case UNION: @@ -173,8 +170,8 @@ public Schema apply(Field f) { } private static boolean visitNonTerminal(final SchemaVisitor visitor, - final Schema schema, final Deque dq, - final Iterable itSupp) { + final Schema schema, final Deque dq, + final Iterable itSupp) { SchemaVisitorAction action = visitor.visitNonTerminal(schema); switch (action) { case CONTINUE: @@ -199,13 +196,9 @@ public SchemaVisitorAction get() { }); break; case SKIP_SIBLINGS: - Object current; - //CHECKSTYLE:OFF InnerAssignment - while ((current = dq.getLast()) instanceof Schema) { - // just skip + while (!dq.isEmpty() && dq.getLast() instanceof Schema) { + dq.removeLast(); } - //CHECKSTYLE:ON - dq.addLast(current); break; case TERMINATE: return true; @@ -216,7 +209,7 @@ public SchemaVisitorAction get() { } private static boolean visitTerminal(final SchemaVisitor visitor, final Schema schema, - final Deque dq) { + final Deque dq) { SchemaVisitorAction action = visitor.visitTerminal(schema); switch (action) { case CONTINUE: @@ -224,13 +217,9 @@ private static boolean visitTerminal(final SchemaVisitor visitor, final Schema s case SKIP_SUBTREE: throw new UnsupportedOperationException("Invalid action " + action + " for " + schema); case SKIP_SIBLINGS: - Object current; - //CHECKSTYLE:OFF InnerAssignment - while ((current = dq.getLast()) instanceof Schema) { - // just skip + while (!dq.isEmpty() && dq.getLast() instanceof Schema) { + dq.removeLast(); } - //CHECKSTYLE:ON - dq.addLast(current); break; case TERMINATE: return true; diff --git a/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/SchemaResolverTest.java b/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/SchemaResolverTest.java deleted file mode 100644 index 61212f7e7e4..00000000000 --- a/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/SchemaResolverTest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2017 The Apache Software Foundation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.avro.compiler.idl; - -import java.io.File; -import java.io.IOException; -import java.net.MalformedURLException; -import org.apache.avro.Protocol; -import org.junit.Assert; -import org.junit.Test; - -/** - * - * @author zoly - */ -public class SchemaResolverTest { - - - @Test - public void testResolving() throws ParseException, MalformedURLException, IOException { - File file = new File("."); - String currentWorkPath = file.getAbsolutePath(); - String testIdl = currentWorkPath + File.separator + "src" + File.separator + "test" - + File.separator + "idl" + File.separator + "cycle.avdl"; - Idl compiler = new Idl(new File(testIdl)); - Protocol protocol = compiler.CompilationUnit(); - System.out.println(protocol); - Assert.assertEquals(5, protocol.getTypes().size()); - } - -} diff --git a/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestCycle.java b/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestCycle.java index bc7695956e1..89a04d6026d 100644 --- a/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestCycle.java +++ b/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestCycle.java @@ -39,14 +39,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * - * @author zoly - */ public class TestCycle { private static final Logger LOG = LoggerFactory.getLogger(TestCycle.class); - + @Test public void testCycleGeneration() throws ParseException, IOException { final ClassLoader cl = Thread.currentThread().getContextClassLoader(); diff --git a/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestSchemaResolver.java b/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestSchemaResolver.java new file mode 100644 index 00000000000..0f7c98f63b6 --- /dev/null +++ b/lang/java/compiler/src/test/java/org/apache/avro/compiler/idl/TestSchemaResolver.java @@ -0,0 +1,72 @@ +/* + * Copyright 2017 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.compiler.idl; + +import java.io.File; +import java.io.IOException; +import java.net.MalformedURLException; + +import org.apache.avro.Protocol; +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; +import org.junit.Assert; +import org.junit.Test; + +public class TestSchemaResolver { + + + @Test + public void testResolving() throws ParseException, MalformedURLException, IOException { + File file = new File("."); + String currentWorkPath = file.getAbsolutePath(); + String testIdl = currentWorkPath + File.separator + "src" + File.separator + "test" + + File.separator + "idl" + File.separator + "cycle.avdl"; + Idl compiler = new Idl(new File(testIdl)); + Protocol protocol = compiler.CompilationUnit(); + System.out.println(protocol); + Assert.assertEquals(5, protocol.getTypes().size()); + } + + @Test(expected = IllegalArgumentException.class) + public void testIsUnresolvedSchemaError1() { + // No "org.apache.avro.compiler.idl.unresolved.name" property + Schema s = SchemaBuilder.record("R").fields().endRecord(); + SchemaResolver.getUnresolvedSchemaName(s); + } + + @Test(expected = IllegalArgumentException.class) + public void testIsUnresolvedSchemaError2() { + // No "UnresolvedSchema" property + Schema s = SchemaBuilder.record("R") + .prop("org.apache.avro.compiler.idl.unresolved.name", "x").fields().endRecord(); + SchemaResolver.getUnresolvedSchemaName(s); + } + + @Test(expected = IllegalArgumentException.class) + public void testIsUnresolvedSchemaError3() { + // Namespace not "org.apache.avro.compiler". + Schema s = SchemaBuilder.record("UnresolvedSchema") + .prop("org.apache.avro.compiler.idl.unresolved.name", "x") + .fields().endRecord(); + SchemaResolver.getUnresolvedSchemaName(s); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetUnresolvedSchemaNameError() { + Schema s = SchemaBuilder.fixed("a").size(10); + SchemaResolver.getUnresolvedSchemaName(s); + } +} diff --git a/lang/java/compiler/src/test/java/org/apache/avro/compiler/schema/SchemasTest.java b/lang/java/compiler/src/test/java/org/apache/avro/compiler/schema/SchemasTest.java deleted file mode 100644 index 545901e53ca..00000000000 --- a/lang/java/compiler/src/test/java/org/apache/avro/compiler/schema/SchemasTest.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright 2017 The Apache Software Foundation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.avro.compiler.schema; - -import org.apache.avro.Schema; -import org.apache.avro.SchemaCompatibility; -import org.junit.Assert; -import org.junit.Test; - -public class SchemasTest { - - private static final String SCHEMA = "{\"type\":\"record\",\"name\":\"SampleNode\",\"doc\":\"caca\"," - + "\"namespace\":\"org.spf4j.ssdump2.avro\",\n" + - " \"fields\":[\n" + - " {\"name\":\"count\",\"type\":\"int\",\"default\":0,\"doc\":\"caca\"},\n" + - " {\"name\":\"subNodes\",\"type\":\n" + - " {\"type\":\"array\",\"items\":{\n" + - " \"type\":\"record\",\"name\":\"SamplePair\",\n" + - " \"fields\":[\n" + - " {\"name\":\"method\",\"type\":\n" + - " {\"type\":\"record\",\"name\":\"Method\",\n" + - " \"fields\":[\n" + - " {\"name\":\"declaringClass\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},\n" + - " {\"name\":\"methodName\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}}\n" + - " ]}},\n" + - " {\"name\":\"node\",\"type\":\"SampleNode\"}]}}}]}"; - - private static class PrintingVisitor implements SchemaVisitor { - - - @Override - public SchemaVisitorAction visitTerminal(Schema terminal) { - System.out.println("Terminal: " + terminal.getFullName()); - return SchemaVisitorAction.CONTINUE; - } - - @Override - public SchemaVisitorAction visitNonTerminal(Schema terminal) { - System.out.println("NONTerminal start: " + terminal.getFullName()); - return SchemaVisitorAction.CONTINUE; - } - - @Override - public SchemaVisitorAction afterVisitNonTerminal(Schema terminal) { - System.out.println("NONTerminal end: " + terminal.getFullName()); - return SchemaVisitorAction.CONTINUE; - } - - @Override - public Object get() { - return null; - } - } - - - - @Test - public void textCloning() { - Schema recSchema = new Schema.Parser().parse(SCHEMA); - Schemas.visit(recSchema, new PrintingVisitor()); - - - Schema trimmed = Schemas.visit(recSchema, new CloningVisitor(recSchema)); - Assert.assertNull(trimmed.getDoc()); - Assert.assertNotNull(recSchema.getDoc()); - - SchemaCompatibility.SchemaCompatibilityType compat = - SchemaCompatibility.checkReaderWriterCompatibility(trimmed, recSchema).getType(); - Assert.assertEquals(SchemaCompatibility.SchemaCompatibilityType.COMPATIBLE, compat); - compat = SchemaCompatibility.checkReaderWriterCompatibility(recSchema, trimmed).getType(); - Assert.assertEquals(SchemaCompatibility.SchemaCompatibilityType.COMPATIBLE, compat); - } - -} diff --git a/lang/java/compiler/src/test/java/org/apache/avro/compiler/schema/TestSchemas.java b/lang/java/compiler/src/test/java/org/apache/avro/compiler/schema/TestSchemas.java new file mode 100644 index 00000000000..230da60a0d4 --- /dev/null +++ b/lang/java/compiler/src/test/java/org/apache/avro/compiler/schema/TestSchemas.java @@ -0,0 +1,347 @@ +/* + * Copyright 2017 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.compiler.schema; + +import org.apache.avro.Schema; +import org.apache.avro.SchemaCompatibility; +import org.junit.Assert; +import org.junit.Test; + +public class TestSchemas { + + private static final String SCHEMA = "{\"type\":\"record\",\"name\":\"SampleNode\",\"doc\":\"caca\"," + + "\"namespace\":\"org.spf4j.ssdump2.avro\",\n" + + " \"fields\":[\n" + + " {\"name\":\"count\",\"type\":\"int\",\"default\":0,\"doc\":\"caca\"},\n" + + " {\"name\":\"kind1\",\"type\":{\"type\":\"enum\", \"name\": \"Kind1\", \"symbols\": [\"A1\", \"B1\"]}},\n" + + " {\"name\":\"kind2\",\"type\":{\"type\":\"enum\", \"name\": \"Kind2\", \"symbols\": [\"A2\", \"B2\"], \"doc\": \"doc\"}},\n" + + " {\"name\":\"pat\",\"type\":{\"type\":\"fixed\", \"name\": \"FixedPattern\", \"size\": 10}},\n" + + " {\"name\":\"uni\",\"type\":[\"int\", \"double\"]},\n" + + " {\"name\":\"mp\",\"type\":{\"type\":\"map\", \"values\": \"int\"}},\n" + + " {\"name\":\"subNodes\",\"type\":\n" + + " {\"type\":\"array\",\"items\":{\n" + + " \"type\":\"record\",\"name\":\"SamplePair\",\n" + + " \"fields\":[\n" + + " {\"name\":\"method\",\"type\":\n" + + " {\"type\":\"record\",\"name\":\"Method\",\n" + + " \"fields\":[\n" + + " {\"name\":\"declaringClass\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},\n" + + " {\"name\":\"methodName\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}}\n" + + " ]}},\n" + + " {\"name\":\"node\",\"type\":\"SampleNode\"}]}}}" + + "]}"; + + private static class PrintingVisitor implements SchemaVisitor { + + + @Override + public SchemaVisitorAction visitTerminal(Schema terminal) { + System.out.println("Terminal: " + terminal.getFullName()); + return SchemaVisitorAction.CONTINUE; + } + + @Override + public SchemaVisitorAction visitNonTerminal(Schema terminal) { + System.out.println("NONTerminal start: " + terminal.getFullName()); + return SchemaVisitorAction.CONTINUE; + } + + @Override + public SchemaVisitorAction afterVisitNonTerminal(Schema terminal) { + System.out.println("NONTerminal end: " + terminal.getFullName()); + return SchemaVisitorAction.CONTINUE; + } + + @Override + public Object get() { + return null; + } + } + + + @Test + public void textCloning() { + Schema recSchema = new Schema.Parser().parse(SCHEMA); + Schemas.visit(recSchema, new PrintingVisitor()); + + + CloningVisitor cv = new CloningVisitor(recSchema); + Schema trimmed = Schemas.visit(recSchema, cv); + Assert.assertNull(trimmed.getDoc()); + Assert.assertNotNull(recSchema.getDoc()); + + SchemaCompatibility.SchemaCompatibilityType compat = + SchemaCompatibility.checkReaderWriterCompatibility(trimmed, recSchema).getType(); + Assert.assertEquals(SchemaCompatibility.SchemaCompatibilityType.COMPATIBLE, compat); + compat = SchemaCompatibility.checkReaderWriterCompatibility(recSchema, trimmed).getType(); + Assert.assertEquals(SchemaCompatibility.SchemaCompatibilityType.COMPATIBLE, compat); + Assert.assertNotNull(cv.toString()); + } + + @Test + public void textCloningCopyDocs() { + Schema recSchema = new Schema.Parser().parse(SCHEMA); + Schemas.visit(recSchema, new PrintingVisitor()); + + + Schema trimmed = Schemas.visit(recSchema, new CloningVisitor(new CloningVisitor.PropertyCopier() { + @Override + public void copy(final Schema first, final Schema second) { + Schemas.copyLogicalTypes(first, second); + Schemas.copyAliases(first, second); + } + + @Override + public void copy(final Schema.Field first, final Schema.Field second) { + Schemas.copyAliases(first, second); + } + }, true, recSchema)); + Assert.assertEquals("caca", trimmed.getDoc()); + Assert.assertNotNull(recSchema.getDoc()); + + SchemaCompatibility.SchemaCompatibilityType compat = + SchemaCompatibility.checkReaderWriterCompatibility(trimmed, recSchema).getType(); + Assert.assertEquals(SchemaCompatibility.SchemaCompatibilityType.COMPATIBLE, compat); + compat = SchemaCompatibility.checkReaderWriterCompatibility(recSchema, trimmed).getType(); + Assert.assertEquals(SchemaCompatibility.SchemaCompatibilityType.COMPATIBLE, compat); + } + + @Test(expected = IllegalStateException.class) + public void testCloningError1() { + // Visit Terminal with union + Schema recordSchema = new Schema.Parser().parse( + "{\"type\": \"record\", \"name\": \"R\", \"fields\":[{\"name\": \"f1\", \"type\": [\"int\", \"long\"]}]}"); + new CloningVisitor(recordSchema).visitTerminal(recordSchema.getField("f1").schema()); + } + + @Test(expected = IllegalStateException.class) + public void testCloningError2() { + // After visit Non-terminal with int + Schema recordSchema = new Schema.Parser().parse( + "{\"type\": \"record\", \"name\": \"R\", \"fields\":[{\"name\": \"f1\", \"type\": \"int\"}]}"); + new CloningVisitor(recordSchema).afterVisitNonTerminal(recordSchema.getField("f1").schema()); + } + + @Test + public void testHasGeneratedJavaClass() { + Assert.assertTrue(Schemas.hasGeneratedJavaClass( + new Schema.Parser().parse("{\"type\": \"fixed\", \"name\": \"N\", \"size\": 10}"))); + Assert.assertFalse(Schemas.hasGeneratedJavaClass(new Schema.Parser().parse("{\"type\": \"int\"}"))); + } + + @Test + public void testGetJavaClassName() { + Assert.assertEquals("N", Schemas.getJavaClassName( + new Schema.Parser().parse("{\"type\": \"fixed\", \"name\": \"N\", \"size\": 10}"))); + Assert.assertEquals("N", Schemas.getJavaClassName( + new Schema.Parser().parse("{\"type\": \"fixed\", \"name\": \"N\", \"size\": 10, \"namespace\": \"\"}"))); + Assert.assertEquals("com.example.N", Schemas.getJavaClassName( + new Schema.Parser().parse("{\"type\": \"fixed\", \"name\": \"N\", \"size\": 10, \"namespace\": \"com.example\"}"))); + } + + private static class TestVisitor implements SchemaVisitor { + StringBuilder sb = new StringBuilder(); + + public SchemaVisitorAction visitTerminal(Schema terminal) { + sb.append(terminal); + return SchemaVisitorAction.CONTINUE; + } + + public SchemaVisitorAction visitNonTerminal(Schema nonTerminal) { + String n = nonTerminal.getName(); + sb.append(n).append('.'); + if (n.startsWith("t")) { + return SchemaVisitorAction.TERMINATE; + } else if (n.startsWith("ss")) { + return SchemaVisitorAction.SKIP_SIBLINGS; + } else if (n.startsWith("st")) { + return SchemaVisitorAction.SKIP_SUBTREE; + } else { + return SchemaVisitorAction.CONTINUE; + } + } + + public SchemaVisitorAction afterVisitNonTerminal(Schema nonTerminal) { + sb.append("!"); + String n = nonTerminal.getName(); + if (n.startsWith("ct")) { + return SchemaVisitorAction.TERMINATE; + } else if (n.startsWith("css")) { + return SchemaVisitorAction.SKIP_SIBLINGS; + } else if (n.startsWith("cst")) { + return SchemaVisitorAction.SKIP_SUBTREE; + } else { + return SchemaVisitorAction.CONTINUE; + } + } + + public String get() { + return sb.toString(); + } + } + + @Test + public void testVisit1() { + String s1 = "{\"type\": \"record\", \"name\": \"t1\", \"fields\": [" + + "{\"name\": \"f1\", \"type\": \"int\"}" + + "]}"; + Assert.assertEquals("t1.", Schemas.visit(new Schema.Parser().parse(s1), new TestVisitor())); + } + + @Test + public void testVisit2() { + String s2 = "{\"type\": \"record\", \"name\": \"c1\", \"fields\": [" + + "{\"name\": \"f1\", \"type\": \"int\"}" + + "]}"; + Assert.assertEquals("c1.\"int\"!", Schemas.visit(new Schema.Parser().parse(s2), new TestVisitor())); + + } + + @Test + public void testVisit3() { + String s3 = "{\"type\": \"record\", \"name\": \"ss1\", \"fields\": [" + + "{\"name\": \"f1\", \"type\": \"int\"}" + + "]}"; + Assert.assertEquals("ss1.", Schemas.visit(new Schema.Parser().parse(s3), new TestVisitor())); + + } + + @Test + public void testVisit4() { + String s4 = "{\"type\": \"record\", \"name\": \"st1\", \"fields\": [" + + "{\"name\": \"f1\", \"type\": \"int\"}" + + "]}"; + Assert.assertEquals("st1.!", Schemas.visit(new Schema.Parser().parse(s4), new TestVisitor())); + + } + + @Test + public void testVisit5() { + String s5 = "{\"type\": \"record\", \"name\": \"c1\", \"fields\": [" + + "{\"name\": \"f1\", \"type\": {\"type\": \"record\", \"name\": \"c2\", \"fields\": " + + "[{\"name\": \"f11\", \"type\": \"int\"}]}}," + + "{\"name\": \"f2\", \"type\": \"long\"}" + + "]}"; + Assert.assertEquals("c1.c2.\"int\"!\"long\"!", + Schemas.visit(new Schema.Parser().parse(s5), new TestVisitor())); + + } + + @Test + public void testVisit6() { + String s6 = "{\"type\": \"record\", \"name\": \"c1\", \"fields\": [" + + "{\"name\": \"f1\", \"type\": {\"type\": \"record\", \"name\": \"ss2\", \"fields\": " + + "[{\"name\": \"f11\", \"type\": \"int\"}]}}," + + "{\"name\": \"f2\", \"type\": \"long\"}" + + "]}"; + Assert.assertEquals("c1.ss2.!", + Schemas.visit(new Schema.Parser().parse(s6), new TestVisitor())); + + } + + @Test + public void testVisit7() { + String s7 = "{\"type\": \"record\", \"name\": \"c1\", \"fields\": [" + + "{\"name\": \"f1\", \"type\": {\"type\": \"record\", \"name\": \"css2\", \"fields\": " + + "[{\"name\": \"f11\", \"type\": \"int\"}]}}," + + "{\"name\": \"f2\", \"type\": \"long\"}" + + "]}"; + Assert.assertEquals("c1.css2.\"int\"!!", + Schemas.visit(new Schema.Parser().parse(s7), new TestVisitor())); + } + + @Test(expected = UnsupportedOperationException.class) + public void testVisit8() { + String s8 = "{\"type\": \"record\", \"name\": \"c1\", \"fields\": [" + + "{\"name\": \"f1\", \"type\": {\"type\": \"record\", \"name\": \"cst2\", \"fields\": " + + "[{\"name\": \"f11\", \"type\": \"int\"}]}}," + + "{\"name\": \"f2\", \"type\": \"int\"}" + + "]}"; + Schemas.visit(new Schema.Parser().parse(s8), new TestVisitor()); + } + + @Test + public void testVisit9() { + String s9 = "{\"type\": \"record\", \"name\": \"c1\", \"fields\": [" + + "{\"name\": \"f1\", \"type\": {\"type\": \"record\", \"name\": \"ct2\", \"fields\": " + + "[{\"name\": \"f11\", \"type\": \"int\"}]}}," + + "{\"name\": \"f2\", \"type\": \"long\"}" + + "]}"; + Assert.assertEquals("c1.ct2.\"int\"!", Schemas.visit(new Schema.Parser().parse(s9), new TestVisitor())); + } + + + @Test(expected = UnsupportedOperationException.class) + public void testVisit10() { + String s10 = "{\"type\": \"record\", \"name\": \"c1\", \"fields\": [" + + "{\"name\": \"f1\", \"type\": {\"type\": \"record\", \"name\": \"ct2\", \"fields\": " + + "[{\"name\": \"f11\", \"type\": \"int\"}]}}," + + "{\"name\": \"f2\", \"type\": \"int\"}" + + "]}"; + Schemas.visit(new Schema.Parser().parse(s10), + new TestVisitor() { + public SchemaVisitorAction visitTerminal(Schema terminal) { + return SchemaVisitorAction.SKIP_SUBTREE; + } + }); + } + + @Test + public void testVisit11() { + String s11 = "{\"type\": \"record\", \"name\": \"c1\", \"fields\": [" + + "{\"name\": \"f1\", \"type\": {\"type\": \"record\", \"name\": \"c2\", \"fields\": " + + "[{\"name\": \"f11\", \"type\": \"int\"},{\"name\": \"f12\", \"type\": \"double\"}" + + "]}}," + + "{\"name\": \"f2\", \"type\": \"long\"}" + + "]}"; + Assert.assertEquals("c1.c2.\"int\".!\"long\".!", Schemas.visit(new Schema.Parser().parse(s11), + new TestVisitor() { + public SchemaVisitorAction visitTerminal(Schema terminal) { + sb.append(terminal).append('.'); + return SchemaVisitorAction.SKIP_SIBLINGS; + } + })); + } + + + @Test + public void testVisit12() { + String s12 = "{\"type\": \"record\", \"name\": \"c1\", \"fields\": [" + + "{\"name\": \"f1\", \"type\": {\"type\": \"record\", \"name\": \"ct2\", \"fields\": " + + "[{\"name\": \"f11\", \"type\": \"int\"}]}}," + + "{\"name\": \"f2\", \"type\": \"long\"}" + + "]}"; + Assert.assertEquals("c1.ct2.\"int\".", Schemas.visit(new Schema.Parser().parse(s12), + new TestVisitor() { + public SchemaVisitorAction visitTerminal(Schema terminal) { + sb.append(terminal).append('.'); + return SchemaVisitorAction.TERMINATE; + } + })); + } + + @Test + public void testVisit13() { + String s12 = "{\"type\": \"int\"}"; + Assert.assertEquals("\"int\".", Schemas.visit(new Schema.Parser().parse(s12), + new TestVisitor() { + public SchemaVisitorAction visitTerminal(Schema terminal) { + sb.append(terminal).append('.'); + return SchemaVisitorAction.SKIP_SIBLINGS; + } + })); + } +} From 53156fd1e475d976df2d55231cbe40b6e0f5dec8 Mon Sep 17 00:00:00 2001 From: "Thiruvalluvan M. G" Date: Mon, 1 May 2017 08:42:21 +0530 Subject: [PATCH 20/22] Added a unit-test for forward referencing of named types --- .../compiler/src/test/idl/input/forward_ref.avdl | 13 +++++++++++++ .../src/test/idl/output/forward_ref.avpr | 16 ++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 lang/java/compiler/src/test/idl/input/forward_ref.avdl create mode 100644 lang/java/compiler/src/test/idl/output/forward_ref.avpr diff --git a/lang/java/compiler/src/test/idl/input/forward_ref.avdl b/lang/java/compiler/src/test/idl/input/forward_ref.avdl new file mode 100644 index 00000000000..a4f108af985 --- /dev/null +++ b/lang/java/compiler/src/test/idl/input/forward_ref.avdl @@ -0,0 +1,13 @@ +@namespace("org.foo") +protocol Import { +/* Name Value record */ +record ANameValue { + /** the name */ + string name; + /** the value */ + string value; + /* is the value a json object */ + ValueType type = "PLAIN"; +} +enum ValueType {JSON, BASE64BIN, PLAIN} +} diff --git a/lang/java/compiler/src/test/idl/output/forward_ref.avpr b/lang/java/compiler/src/test/idl/output/forward_ref.avpr new file mode 100644 index 00000000000..06531b55479 --- /dev/null +++ b/lang/java/compiler/src/test/idl/output/forward_ref.avpr @@ -0,0 +1,16 @@ +{ + "protocol": "Import", + "namespace": "org.foo", + "types": [ + { + "type": "record", + "name": "ANameValue", + "fields": [ + { "name":"name", "type": "string", "doc":"the name" }, + { "name": "value", "type": "string", "doc": "the value" }, + { "name": "type", "type": { "type": "enum", "name":"ValueType", "symbols": ["JSON","BASE64BIN","PLAIN"] } } + ] + } + ], + "messages": { } +} From 5795c5e505b817d6481eb1863068c0aa0ad329b9 Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Sun, 14 May 2017 19:44:47 -0400 Subject: [PATCH 21/22] [fix] redo guava + shading in the compiler. --- lang/java/compiler/pom.xml | 26 ++++++++++++++++++- .../avro/compiler/idl/ResolvingVisitor.java | 2 +- .../avro/compiler/idl/SchemaResolver.java | 3 +-- .../apache/avro/compiler/schema/Schemas.java | 7 +++-- 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/lang/java/compiler/pom.xml b/lang/java/compiler/pom.xml index 3ff010197f7..912e09a4b27 100644 --- a/lang/java/compiler/pom.xml +++ b/lang/java/compiler/pom.xml @@ -89,6 +89,31 @@ + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + + + + org.apache.avro:avro-guava-dependencies + + + + + com.google.common + avro.shaded.com.google.common + + + + + + + @@ -137,7 +162,6 @@ com.google.guava guava ${guava.version} - test com.google.code.findbugs diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/ResolvingVisitor.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/ResolvingVisitor.java index 35fb44b30a0..5323b183d19 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/ResolvingVisitor.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/ResolvingVisitor.java @@ -1,7 +1,7 @@ package org.apache.avro.compiler.idl; -import avro.shaded.com.google.common.base.Function; +import com.google.common.base.Function; import java.util.ArrayList; import java.util.IdentityHashMap; import java.util.List; diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java index 596523ca0b0..0195df452ee 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/idl/SchemaResolver.java @@ -15,8 +15,7 @@ */ package org.apache.avro.compiler.idl; -import avro.shaded.com.google.common.base.Function; - +import com.google.common.base.Function; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/Schemas.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/Schemas.java index ac39d883ecb..3e8fee06b44 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/Schemas.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/schema/Schemas.java @@ -1,9 +1,8 @@ package org.apache.avro.compiler.schema; -import avro.shaded.com.google.common.base.Function; -import avro.shaded.com.google.common.base.Supplier; -import avro.shaded.com.google.common.collect.Lists; - +import com.google.common.base.Function; +import com.google.common.base.Supplier; +import com.google.common.collect.Lists; import java.util.ArrayDeque; import java.util.Arrays; import java.util.Deque; From 9d76342ae8d7d017b0fb71c673bf5774b7bf6f61 Mon Sep 17 00:00:00 2001 From: Zoltan Farkas Date: Sun, 14 May 2017 20:37:40 -0400 Subject: [PATCH 22/22] [fix] redo guava + shading in the compiler. --- lang/java/compiler/pom.xml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/lang/java/compiler/pom.xml b/lang/java/compiler/pom.xml index 912e09a4b27..ee260c7f14b 100644 --- a/lang/java/compiler/pom.xml +++ b/lang/java/compiler/pom.xml @@ -159,15 +159,9 @@ joda-time - com.google.guava - guava - ${guava.version} - - - com.google.code.findbugs - jsr305 - - + org.apache.avro + avro-guava-dependencies + ${project.version}