Skip to content

Commit

Permalink
Support for docs in schemas.
Browse files Browse the repository at this point in the history
  • Loading branch information
Philip Zeyliger committed Oct 16, 2009
1 parent efc086a commit b00e1a3
Show file tree
Hide file tree
Showing 8 changed files with 148 additions and 45 deletions.
10 changes: 9 additions & 1 deletion src/doc/content/xdocs/spec.xml
View file
Expand Up @@ -86,15 +86,19 @@
<section id="schema_record">
<title>Records</title>

<p>Records use the type name "record" and support two attributes:</p>
<p>Records use the type name "record" and support three attributes:</p>
<ul>
<li><code>name</code>: a JSON string providing the name
of the record (required).</li>
<li><code>doc</code>: a JSON string providing documentation to the
user of this schema (optional).</li>
<li><code>fields</code>: a JSON array, listing fields (required).
Each field is a JSON object with the following attributes:
<ul>
<li><code>name</code>: a JSON string providing the name
of the field (required), and </li>
<li><code>doc</code>: a JSON string describing this field
for users (optional).</li>
<li><code>type:</code> A JSON object defining a schema, or
a JSON string naming a record definition
(required).</li>
Expand Down Expand Up @@ -977,6 +981,10 @@
</li>

</ul>

<p>A schema's "doc" fields are ignored for the purposes of schema resolution. Hence,
the "doc" portion of a schema may be dropped at serialization.</p>

</section>
</body>
</document>
1 change: 1 addition & 0 deletions src/java/org/apache/avro/Protocol.java
View file
Expand Up @@ -318,6 +318,7 @@ private Message parseMessage(String messageName, JsonNode json) {
throw new SchemaParseException("No param type: "+field);
fields.put(fieldNameNode.getTextValue(),
new Field(Schema.parse(fieldTypeNode,types),
null, /* TODO(philip) */
field.get("default")));
}
Schema request = Schema.createRecord(fields);
Expand Down
101 changes: 69 additions & 32 deletions src/java/org/apache/avro/Schema.java
View file
Expand Up @@ -94,21 +94,21 @@ public static Schema create(Type type) {

/** Create an anonymous record schema. */
public static Schema createRecord(LinkedHashMap<String,Field> fields) {
Schema result = createRecord(null, null, false);
Schema result = createRecord(null, null, null, false);
result.setFields(fields);
return result;
}

/** Create a named record schema. */
public static Schema createRecord(String name, String namespace,
public static Schema createRecord(String name, String doc, String namespace,
boolean isError) {
return new RecordSchema(name, namespace, isError);
return new RecordSchema(name, doc, namespace, isError);
}

/** Create an enum schema. */
public static Schema createEnum(String name, String namespace,
public static Schema createEnum(String name, String doc, String namespace,
List<String> values) {
return new EnumSchema(name, namespace, values);
return new EnumSchema(name, doc, namespace, values);
}

/** Create an array schema. */
Expand All @@ -127,8 +127,8 @@ public static Schema createUnion(List<Schema> types) {
}

/** Create a union schema. */
public static Schema createFixed(String name, String space, int size) {
return new FixedSchema(name, space, size);
public static Schema createFixed(String name, String doc, String space, int size) {
return new FixedSchema(name, doc, space, size);
}

/** Return the type of this schema. */
Expand Down Expand Up @@ -160,8 +160,16 @@ public int getEnumOrdinal(String symbol) {
}

/** If this is a record, enum or fixed, returns its name, otherwise the name
* of the primitive type. */
* of the primitive type. Never returns null. */
public String getName() { return type.name; }

/**
* If this is a record, enum, or fixed, returns its docstring,
* if available. Otherwise, returns null.
*/
public String getDoc() {
return null;
}

/** If this is a record, enum or fixed, returns its namespace, if any. */
public String getNamespace() {
Expand Down Expand Up @@ -233,21 +241,27 @@ public enum Order {

private int position = -1;
private final Schema schema;
private final String doc;
private final JsonNode defaultValue;
private final Order order;

public Field(Schema schema, JsonNode defaultValue) {
this(schema, defaultValue, Order.ASCENDING);
public Field(Schema schema, String doc, JsonNode defaultValue) {
this(schema, doc, defaultValue, Order.ASCENDING);
}
public Field(Schema schema, JsonNode defaultValue, Order order) {
public Field(Schema schema, String doc, JsonNode defaultValue, Order order) {
this.schema = schema;
this.doc = doc;
this.defaultValue = defaultValue;
this.order = order;
}
/** The position of this field within the record. */
public int pos() { return position; }
/** This field's {@link Schema}. */
public Schema schema() { return schema; }
/**
* This field's documentation within the record, if set. May return null.
*/
public String doc() { return doc; }
public JsonNode defaultValue() { return defaultValue; }
public Order order() { return order; }
public boolean equals(Object other) {
Expand Down Expand Up @@ -303,11 +317,14 @@ public void writeName(Names names, JsonGenerator gen) throws IOException {

private static abstract class NamedSchema extends Schema {
private final Name name;
public NamedSchema(Type type, String name, String space) {
private final String doc;
public NamedSchema(Type type, String name, String doc, String space) {
super(type);
this.name = new Name(name, space);
this.doc = doc;
}
public String getName() { return name.name; }
public String getDoc() { return doc; }
public String getNamespace() { return name.space; }
public boolean writeNameRef(Names names, JsonGenerator gen)
throws IOException {
Expand Down Expand Up @@ -355,8 +372,8 @@ private static class RecordSchema extends NamedSchema {
private Map<String,Field> fields;
private Iterable<Map.Entry<String,Schema>> fieldSchemas;
private final boolean isError;
public RecordSchema(String name, String space, boolean isError) {
super(Type.RECORD, name, space);
public RecordSchema(String name, String doc, String space, boolean isError) {
super(Type.RECORD, name, doc, space);
this.isError = isError;
}
public boolean isError() { return isError; }
Expand Down Expand Up @@ -437,8 +454,8 @@ void fieldsToJson(Names names, JsonGenerator gen) throws IOException {
private static class EnumSchema extends NamedSchema {
private final List<String> symbols;
private final Map<String,Integer> ordinals;
public EnumSchema(String name, String space, List<String> symbols) {
super(Type.ENUM, name, space);
public EnumSchema(String name, String doc, String space, List<String> symbols) {
super(Type.ENUM, name, doc, space);
this.symbols = symbols;
this.ordinals = new HashMap<String,Integer>();
int i = 0;
Expand Down Expand Up @@ -550,8 +567,8 @@ void toJson(Names names, JsonGenerator gen) throws IOException {

private static class FixedSchema extends NamedSchema {
private final int size;
public FixedSchema(String name, String space, int size) {
super(Type.FIXED, name, space);
public FixedSchema(String name, String doc, String space, int size) {
super(Type.FIXED, name, doc, space);
this.size = size;
}
public int getFixedSize() { return size; }
Expand Down Expand Up @@ -693,13 +710,16 @@ static Schema parse(JsonNode schema, Names names) {
if (typeNode == null)
throw new SchemaParseException("No type: "+schema);
String type = typeNode.getTextValue();
String name = null, space = null;
String name = null, space = null, doc = null;
if (type.equals("record") || type.equals("error")
|| type.equals("enum") || type.equals("fixed")) {
JsonNode nameNode = schema.get("name");
name = nameNode != null ? nameNode.getTextValue() : null;
JsonNode spaceNode = schema.get("namespace");
space = spaceNode!=null?spaceNode.getTextValue():names.space();
String key = "name";
name = getStringValueOrNull(schema, "name");
doc = getStringValueOrNull(schema, "doc");
space = getStringValueOrNull(schema, "namespace");
if (space == null) {
space = names.space();
}
if (names.space() == null && space != null)
names.space(space); // set default namespace
if (name == null)
Expand All @@ -708,15 +728,17 @@ static Schema parse(JsonNode schema, Names names) {
if (type.equals("record") || type.equals("error")) { // record
LinkedHashMap<String,Field> fields = new LinkedHashMap<String,Field>();
RecordSchema result =
new RecordSchema(name, space, type.equals("error"));
new RecordSchema(name, doc, space, type.equals("error"));
if (name != null) names.add(result);
JsonNode fieldsNode = schema.get("fields");
if (fieldsNode == null || !fieldsNode.isArray())
throw new SchemaParseException("Record has no fields: "+schema);
for (JsonNode field : fieldsNode) {
JsonNode fieldNameNode = field.get("name");
if (fieldNameNode == null)
String fieldName = getStringValueOrNull(field, "name");
String fieldDoc = getStringValueOrNull(field, "doc");
if (fieldName == null) {
throw new SchemaParseException("No field name: "+field);
}
JsonNode fieldTypeNode = field.get("type");
if (fieldTypeNode == null)
throw new SchemaParseException("No field type: "+field);
Expand All @@ -725,8 +747,8 @@ static Schema parse(JsonNode schema, Names names) {
JsonNode orderNode = field.get("order");
if (orderNode != null)
order = Field.Order.valueOf(orderNode.getTextValue().toUpperCase());
fields.put(fieldNameNode.getTextValue(),
new Field(fieldSchema, field.get("default"), order));
fields.put(fieldName,
new Field(fieldSchema, fieldDoc, field.get("default"), order));
}
result.setFields(fields);
return result;
Expand All @@ -737,17 +759,24 @@ static Schema parse(JsonNode schema, Names names) {
List<String> symbols = new ArrayList<String>();
for (JsonNode n : symbolsNode)
symbols.add(n.getTextValue());
Schema result = new EnumSchema(name, space, symbols);
Schema result = new EnumSchema(name, doc, space, symbols);
if (name != null) names.add(result);
return result;
} else if (type.equals("array")) { // array
return new ArraySchema(parse(schema.get("items"), names));
} else if (type.equals("map")) { // map
return new MapSchema(parse(schema.get("values"), names));
} else if (type.equals("fixed")) { // fixed
Schema result = new FixedSchema(name, space,
schema.get("size").getIntValue());
if (name != null) names.add(result);
JsonNode sizeNode = schema.get("size");
if (sizeNode == null || !sizeNode.isInt()) {
throw new SchemaParseException(
"Fixed node has no or non-integer size: " + schema);
}
Schema result = new FixedSchema(name, doc, space,
sizeNode.getIntValue());
if (name != null) {
names.add(result);
}
return result;
} else
throw new SchemaParseException("Type not yet supported: "+type);
Expand All @@ -761,6 +790,14 @@ static Schema parse(JsonNode schema, Names names) {
}
}

/**
* Extracts text value associated to key from the container JsonNode.
*/
private static String getStringValueOrNull(JsonNode container, String key) {
JsonNode jsonNode = container.get(key);
return jsonNode != null ? jsonNode.getTextValue() : null;
}

static JsonNode parseJson(String s) {
try {
return MAPPER.readTree(FACTORY.createJsonParser(new StringReader(s)));
Expand Down
4 changes: 2 additions & 2 deletions src/java/org/apache/avro/generic/GenericData.java
View file
Expand Up @@ -267,7 +267,7 @@ public Schema induce(Object datum) {
GenericRecord record = (GenericRecord)datum;
LinkedHashMap<String,Field> fields = new LinkedHashMap<String,Field>();
for (Map.Entry<String,Object> entry : record.entrySet())
fields.put(entry.getKey(), new Field(induce(entry.getValue()), null));
fields.put(entry.getKey(), new Field(induce(entry.getValue()), null, null));
return Schema.createRecord(fields);
} else if (datum instanceof GenericArray) {
Schema elementType = null;
Expand Down Expand Up @@ -299,7 +299,7 @@ public Schema induce(Object datum) {
}
return Schema.createMap(value);
} else if (datum instanceof GenericFixed) {
return Schema.createFixed(null, null,
return Schema.createFixed(null, null, null,
((GenericFixed)datum).bytes().length);
}
else if (datum instanceof Utf8) return Schema.create(Type.STRING);
Expand Down
13 changes: 7 additions & 6 deletions src/java/org/apache/avro/reflect/ReflectData.java
View file
Expand Up @@ -217,7 +217,8 @@ public Schema getSchema(java.lang.reflect.Type type) {
/**
* Create a schema for a type and it's fields. Note that by design only fields
* of the direct class, not it's super classes, are used for creating the
* schema. Also, fields are not permitted to be null.
* schema. Also, fields are not permitted to be null. Javadoc is unavailable
* via reflection, so the "doc" field of the schema is simply null.
*/
@SuppressWarnings(value="unchecked")
protected Schema createSchema(java.lang.reflect.Type type,
Expand Down Expand Up @@ -269,28 +270,28 @@ else if (type instanceof ParameterizedType) {
Enum[] constants = (Enum[])c.getEnumConstants();
for (int i = 0; i < constants.length; i++)
symbols.add(constants[i].name());
schema = Schema.createEnum(name, space, symbols);
schema = Schema.createEnum(name, null /* doc */, space, symbols);
names.put(fullName, schema);
return schema;
}
// fixed
if (GenericFixed.class.isAssignableFrom(c)) {
int size = ((FixedSize)c.getAnnotation(FixedSize.class)).value();
schema = Schema.createFixed(name, space, size);
schema = Schema.createFixed(name, null /* doc */, space, size);
names.put(fullName, schema);
return schema;
}
// record
LinkedHashMap<String,Schema.Field> fields =
new LinkedHashMap<String,Schema.Field>();
schema = Schema.createRecord(name, space,
schema = Schema.createRecord(name, null /* doc */, space,
Throwable.class.isAssignableFrom(c));
if (!names.containsKey(fullName))
names.put(fullName, schema);
for (Field field : c.getDeclaredFields())
if ((field.getModifiers()&(Modifier.TRANSIENT|Modifier.STATIC))==0) {
Schema fieldSchema = createFieldSchema(field, names);
fields.put(field.getName(), new Schema.Field(fieldSchema, null));
fields.put(field.getName(), new Schema.Field(fieldSchema, null, null));
}
schema.setFields(fields);
}
Expand Down Expand Up @@ -338,7 +339,7 @@ private Message getMessage(Method method, Protocol protocol,
java.lang.reflect.Type[] paramTypes = method.getGenericParameterTypes();
for (int i = 0; i < paramTypes.length; i++)
fields.put(paramNames[i],
new Schema.Field(createSchema(paramTypes[i], names), null));
new Schema.Field(createSchema(paramTypes[i], names), null, null));
Schema request = Schema.createRecord(fields);

Schema response = createSchema(method.getGenericReturnType(), names);
Expand Down

0 comments on commit b00e1a3

Please sign in to comment.