Skip to content
Browse files

AVRO-1103. Java: Fix SpecificData and mapreduce to use correct classl…

…oader.

git-svn-id: https://svn.apache.org/repos/asf/avro/trunk@1368249 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information...
1 parent be2f378 commit 196011b43149f38797fb7c462a0a46e0263d9f4e @cutting cutting committed Aug 1, 2012
View
3 CHANGES.txt
@@ -14,6 +14,9 @@ Avro 1.7.2 (unreleased)
AVRO-1131. Generated build makefiles for MSYS/MinGW use Visual
Studio compiler flags (Laurent Moss via thiru)
+ AVRO-1103. Java: Fix SpecificData and mapreduce to use correct
+ classloader. (cutting)
+
Avro 1.7.1 (16 July 2012)
NEW FEATURES
View
5 lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java
@@ -76,6 +76,11 @@ protected Schema createFieldSchema(Field field, Map<String, Schema> names) {
protected ReflectData() {}
+ /** Construct with a particular classloader. */
+ public ReflectData(ClassLoader classLoader) {
+ super(classLoader);
+ }
+
/** Return the singleton instance. */
public static ReflectData get() { return INSTANCE; }
View
3 lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectDatumReader.java
@@ -52,7 +52,8 @@ public ReflectDatumReader(Schema writer, Schema reader) {
this(writer, reader, ReflectData.get());
}
- protected ReflectDatumReader(Schema writer, Schema reader, ReflectData data){
+ /** Construct given writer's and reader's schema and the data model. */
+ public ReflectDatumReader(Schema writer, Schema reader, ReflectData data) {
super(writer, reader, data);
}
View
7 lang/java/avro/src/main/java/org/apache/avro/specific/SpecificData.java
@@ -55,6 +55,9 @@ public SpecificData(ClassLoader classLoader) {
this.classLoader = classLoader;
}
+ /** Return the class loader that's used. */
+ public ClassLoader getClassLoader() { return classLoader; }
+
@Override
public DatumReader createDatumReader(Schema schema) {
return new SpecificDatumReader(schema, schema, this);
@@ -267,14 +270,14 @@ public static Object newInstance(Class c, Schema s) {
@Override
public Object createFixed(Object old, Schema schema) {
- Class c = SpecificData.get().getClass(schema);
+ Class c = getClass(schema);
if (c == null) return super.createFixed(old, schema); // punt to generic
return c.isInstance(old) ? old : newInstance(c, schema);
}
@Override
public Object newRecord(Object old, Schema schema) {
- Class c = SpecificData.get().getClass(schema);
+ Class c = getClass(schema);
if (c == null) return super.newRecord(old, schema); // punt to generic
return (c.isInstance(old) ? old : newInstance(c, schema));
}
View
9 lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroDeserializer.java
@@ -26,6 +26,7 @@
import org.apache.avro.io.BinaryDecoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.mapred.AvroWrapper;
+import org.apache.avro.reflect.ReflectData;
import org.apache.avro.reflect.ReflectDatumReader;
import org.apache.hadoop.io.serializer.Deserializer;
@@ -52,7 +53,7 @@
private final Schema mReaderSchema;
/** The Avro datum reader for deserializing. */
- private final DatumReader<D> mAvroDatumReader;
+ final DatumReader<D> mAvroDatumReader;
/** An Avro binary decoder for deserializing. */
private BinaryDecoder mAvroDecoder;
@@ -63,10 +64,12 @@
* @param writerSchema The Avro writer schema for the data to deserialize.
* @param readerSchema The Avro reader schema for the data to deserialize (may be null).
*/
- protected AvroDeserializer(Schema writerSchema, Schema readerSchema) {
+ protected AvroDeserializer(Schema writerSchema, Schema readerSchema,
+ ClassLoader classLoader) {
mWriterSchema = writerSchema;
mReaderSchema = null != readerSchema ? readerSchema : writerSchema;
- mAvroDatumReader = new ReflectDatumReader<D>(mWriterSchema, mReaderSchema);
+ mAvroDatumReader = new ReflectDatumReader<D>(mWriterSchema, mReaderSchema,
+ new ReflectData(classLoader));
}
/**
View
5 lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroKeyDeserializer.java
@@ -36,8 +36,9 @@
* @param writerSchema The Avro writer schema for the data to deserialize.
* @param readerSchema The Avro reader schema for the data to deserialize.
*/
- public AvroKeyDeserializer(Schema writerSchema, Schema readerSchema) {
- super(writerSchema, readerSchema);
+ public AvroKeyDeserializer(Schema writerSchema, Schema readerSchema,
+ ClassLoader classLoader) {
+ super(writerSchema, readerSchema, classLoader);
}
/**
View
8 lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroSerialization.java
@@ -65,9 +65,13 @@ public boolean accept(Class<?> c) {
public Deserializer<AvroWrapper<T>> getDeserializer(Class<AvroWrapper<T>> c) {
Configuration conf = getConf();
if (AvroKey.class.isAssignableFrom(c)) {
- return new AvroKeyDeserializer<T>(getKeyWriterSchema(conf), getKeyReaderSchema(conf));
+ return new AvroKeyDeserializer<T>(getKeyWriterSchema(conf),
+ getKeyReaderSchema(conf),
+ conf.getClassLoader());
} else if (AvroValue.class.isAssignableFrom(c)) {
- return new AvroValueDeserializer<T>(getValueWriterSchema(conf), getValueReaderSchema(conf));
+ return new AvroValueDeserializer<T>(getValueWriterSchema(conf),
+ getValueReaderSchema(conf),
+ conf.getClassLoader());
} else {
throw new IllegalStateException("Only AvroKey and AvroValue are supported.");
}
View
5 lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroValueDeserializer.java
@@ -36,8 +36,9 @@
* @param writerSchema The Avro writer schema for the data to deserialize.
* @param readerSchema The Avro reader schema for the data to deserialize.
*/
- public AvroValueDeserializer(Schema writerSchema, Schema readerSchema) {
- super(writerSchema, readerSchema);
+ public AvroValueDeserializer(Schema writerSchema, Schema readerSchema,
+ ClassLoader classLoader) {
+ super(writerSchema, readerSchema, classLoader);
}
/**
View
4 lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroKeyDeserializer.java
@@ -38,8 +38,10 @@ public void testDeserialize() throws IOException {
// Create a deserializer.
Schema writerSchema = Schema.create(Schema.Type.STRING);
Schema readerSchema = Schema.create(Schema.Type.STRING);
+ ClassLoader classLoader = this.getClass().getClassLoader();
AvroKeyDeserializer<CharSequence> deserializer
- = new AvroKeyDeserializer<CharSequence>(writerSchema, readerSchema);
+ = new AvroKeyDeserializer<CharSequence>(writerSchema, readerSchema,
+ classLoader);
// Check the schemas.
assertEquals(writerSchema, deserializer.getWriterSchema());
View
17 lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroSerialization.java
@@ -22,15 +22,19 @@
import java.io.IOException;
import org.apache.avro.Schema;
+import org.apache.avro.reflect.ReflectData;
+import org.apache.avro.reflect.ReflectDatumReader;
import org.apache.avro.mapred.AvroKey;
import org.apache.avro.mapred.AvroValue;
import org.apache.avro.mapred.AvroWrapper;
import org.apache.avro.mapreduce.AvroJob;
import org.apache.hadoop.io.serializer.Deserializer;
import org.apache.hadoop.io.serializer.Serializer;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ReflectionUtils;
import org.junit.Test;
+import org.junit.Assert;
public class TestAvroSerialization {
@Test
@@ -118,4 +122,17 @@ public void testGetDeserializerForValue() throws IOException {
// Check that the reader schema is set correctly on the deserializer.
assertEquals(readerSchema, avroDeserializer.getReaderSchema());
}
+
+ @Test public void testClassPath() throws Exception {
+ Configuration conf = new Configuration();
+ ClassLoader loader = new ClassLoader() {};
+ conf.setClassLoader(loader);
+ AvroSerialization serialization = new AvroSerialization();
+ serialization.setConf(conf);
+ AvroDeserializer des =
+ (AvroDeserializer)serialization.getDeserializer(AvroKey.class);
+ ReflectData data =
+ (ReflectData)((ReflectDatumReader)des.mAvroDatumReader).getData();
+ Assert.assertEquals(loader, data.getClassLoader());
+ }
}
View
4 lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroValueDeserializer.java
@@ -38,8 +38,10 @@ public void testDeserialize() throws IOException {
// Create a deserializer.
Schema writerSchema = Schema.create(Schema.Type.STRING);
Schema readerSchema = Schema.create(Schema.Type.STRING);
+ ClassLoader classLoader = this.getClass().getClassLoader();
AvroValueDeserializer<CharSequence> deserializer
- = new AvroValueDeserializer<CharSequence>(writerSchema, readerSchema);
+ = new AvroValueDeserializer<CharSequence>(writerSchema, readerSchema,
+ classLoader);
// Check the schemas.
assertEquals(writerSchema, deserializer.getWriterSchema());

0 comments on commit 196011b

Please sign in to comment.
Something went wrong with that request. Please try again.